2020-12-28 13:13:24 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 runnable [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #847 daemon prio=10 os_prio=0 tid=0x00007ef3e005b000 nid=0x14d7 waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c6674b0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdb049f578> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9311a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4c4268> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe78260ce0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bb2b880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdaa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff6e761580> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd506b2c28> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b17ec68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1070" #621 prio=5 os_prio=0 tid=0x00007ef65e5db800 nid=0x8af7 runnable [0x00007ef7ee7fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bbad0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 waiting for monitor entry [0x00007ef7ee9fe000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe916beba8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effaea51508> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efee3d1aa70> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c930078> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839c8628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b36cf30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff7488c428> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe804ea878> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c022990> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe73d18d38> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a5000a8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913427c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04467e0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04467c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0441ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effae71d7b0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78068a68> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d04a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdae440078> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 waiting for monitor entry [0x00007ef8d3135000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - locked <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c229448> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff912b8dd8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdae3a8330> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff1b9ae268> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5ed4dc00> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf0160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff144671c8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78c80478> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b923c78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0818e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae0818c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae07df48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c632650> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1071" #485 prio=5 os_prio=0 tid=0x00007ef65d78f000 nid=0x8a6d runnable [0x00007ef9561fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528b30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a2b48e0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe6ebd0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe78077ce8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efda9cdf438> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed9e865d8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5517dd60> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80dc89d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdafbe4c08> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb82a7178> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe80992a68> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7dc701b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe93d9fc90> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdafc8ddd0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efec8a9d118> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe809e0100> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa4ab8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff044be040> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d36ab30> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b waiting for monitor entry [0x00007efa7083b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910b4128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0f4d810> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effaf20eab0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd59e99790> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae065670> (a sun.nio.ch.Util$3) - locked <0x00007efdae065658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae060950> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:477) at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:450) at org.apache.kafka.clients.NetworkClient.send(NetworkClient.java:411) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:91) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa17a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe8d8c1dc0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5ea86e88> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad22cb38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7ac929b8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad50e370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:43) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb075d8c0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe4c50aa30> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff6c045cd0> (a java.lang.Object) - locked <0x00007eff6c045cb0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff4a138178> (a java.lang.Object) - locked <0x00007eff4a138158> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb5ff70f8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a316e00> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac79d3a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efef1101520> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd550ca710> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c86e598> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac644d50> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff1b8619e8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd58fbd6e8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae24f450> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe8cce2770> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd82fef538> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 waiting on condition [0x00007efbff5f4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 waiting on condition [0x00007efbff6f5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff8ce0c008> (a java.lang.Object) - locked <0x00007eff8ce0bfe8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a waiting on condition [0x00007efc06d5a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: RUNNABLE at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:73) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 waiting on condition [0x00007efc0705d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 waiting on condition [0x00007efc0889e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 waiting on condition [0x00007efc09208000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c229448> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 waiting on condition [0x00007efc60c84000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: RUNNABLE at java.lang.Throwable.fillInStackTrace(Native Method) at java.lang.Throwable.fillInStackTrace(Throwable.java:783) - locked <0x00007f0534c756b0> (a java.io.IOException) at java.lang.Throwable.(Throwable.java:265) at java.lang.Exception.(Exception.java:66) at java.io.IOException.(IOException.java:58) at org.apache.hadoop.security.Groups.noGroupsForUser(Groups.java:200) at org.apache.hadoop.security.Groups.getGroups(Groups.java:223) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.SessionlessFetchContext$$anonfun$foreachPartition$1.apply(FetchSession.scala:331) at kafka.server.SessionlessFetchContext$$anonfun$foreachPartition$1.apply(FetchSession.scala:331) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.SessionlessFetchContext.foreachPartition(FetchSession.scala:331) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: RUNNABLE at kafka.server.ReplicaManager.getLeaderReplicaIfLocal(ReplicaManager.scala:420) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$read$1(ReplicaManager.scala:896) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:989) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:988) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:988) at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:815) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:828) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) - locked <0x00007efd86746a58> (a java.lang.Object) at org.apache.kafka.common.network.Selector.wakeup(Selector.java:305) at kafka.network.Processor.wakeup(SocketServer.scala:870) at kafka.network.Processor.enqueueResponse(SocketServer.scala:829) at kafka.network.RequestChannel.sendResponse(RequestChannel.scala:336) at kafka.server.KafkaApis.kafka$server$KafkaApis$$sendResponse(KafkaApis.scala:2362) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2358) at kafka.server.KafkaApis.sendResponseMaybeThrottle(KafkaApis.scala:2300) at kafka.server.KafkaApis.handleTopicMetadataRequest(KafkaApis.scala:1092) at kafka.server.KafkaApis.handle(KafkaApis.scala:109) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e waiting on condition [0x00007ef5f8dfe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d waiting on condition [0x00007ef603e76000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c waiting on condition [0x00007ef5f03fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 waiting on condition [0x00007ef532e13000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.MetadataCache.getPartitionInfo(MetadataCache.scala:178) at kafka.server.MetadataCache.contains(MetadataCache.scala:277) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:537) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007efe39e2da68> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.requests.MetadataResponse.toStruct(MetadataResponse.java:559) at org.apache.kafka.common.requests.AbstractResponse.serialize(AbstractResponse.java:41) at org.apache.kafka.common.requests.AbstractResponse.toSend(AbstractResponse.java:34) at org.apache.kafka.common.requests.RequestContext.buildResponse(RequestContext.java:78) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2350) at kafka.server.KafkaApis.sendResponseMaybeThrottle(KafkaApis.scala:2300) at kafka.server.KafkaApis.handleTopicMetadataRequest(KafkaApis.scala:1092) at kafka.server.KafkaApis.handle(KafkaApis.scala:109) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 waiting on condition [0x00007ef53851b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 waiting on condition [0x00007efbe5bd9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 waiting on condition [0x00007ef530355000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f waiting on condition [0x00007ef538248000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e waiting on condition [0x00007efae78e1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d waiting on condition [0x00007ef600175000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a runnable [0x00007efb00172000] java.lang.Thread.State: RUNNABLE at java.io.FileOutputStream.open0(Native Method) at java.io.FileOutputStream.open(FileOutputStream.java:270) at java.io.FileOutputStream.(FileOutputStream.java:213) at java.io.FileOutputStream.(FileOutputStream.java:162) at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:52) at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50) - locked <0x00007efd507a6190> (a java.lang.Object) at kafka.server.checkpoints.LeaderEpochCheckpointFile.write(LeaderEpochCheckpointFile.scala:64) at kafka.server.epoch.LeaderEpochFileCache.kafka$server$epoch$LeaderEpochFileCache$$flush(LeaderEpochFileCache.scala:193) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$clearAndFlushEarliest$1.apply$mcV$sp(LeaderEpochFileCache.scala:157) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$clearAndFlushEarliest$1.apply(LeaderEpochFileCache.scala:148) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$clearAndFlushEarliest$1.apply(LeaderEpochFileCache.scala:148) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.clearAndFlushEarliest(LeaderEpochFileCache.scala:148) at kafka.log.Log$$anonfun$maybeIncrementLogStartOffset$1.apply$mcV$sp(Log.scala:945) - locked <0x00007efd5078e1d0> (a java.lang.Object) at kafka.log.Log$$anonfun$maybeIncrementLogStartOffset$1.apply(Log.scala:940) at kafka.log.Log$$anonfun$maybeIncrementLogStartOffset$1.apply(Log.scala:940) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.maybeIncrementLogStartOffset(Log.scala:939) at kafka.log.Log$$anonfun$deleteSegments$1.apply$mcI$sp(Log.scala:1312) - locked <0x00007efd5078e1d0> (a java.lang.Object) at kafka.log.Log$$anonfun$deleteSegments$1.apply(Log.scala:1302) at kafka.log.Log$$anonfun$deleteSegments$1.apply(Log.scala:1302) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.deleteSegments(Log.scala:1302) at kafka.log.Log.deleteOldSegments(Log.scala:1297) - locked <0x00007efd5078e1d0> (a java.lang.Object) at kafka.log.Log.deleteRetentionMsBreachedSegments(Log.scala:1369) at kafka.log.Log.deleteOldSegments(Log.scala:1363) at kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:874) at kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:872) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.immutable.List.foreach(List.scala:392) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.LogManager.cleanupLogs(LogManager.scala:872) at kafka.log.LogManager$$anonfun$startup$1.apply$mcV$sp(LogManager.scala:395) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506d7910> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00007efd507a61e0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 runnable [0x00007efb141d5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efd5078e1d0> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 runnable [0x00007efb246ae000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting for monitor entry [0x00007efb3c1e2000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.logSegments(Log.scala:1711) - waiting to lock <0x00007efd5078e1d0> (a java.lang.Object) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506d7b50> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 waiting on condition [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.sanitizeName(KafkaTimelineMetricsReporter.java:502) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:414) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:13:36 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "Keep-Alive-Timer" #915 daemon prio=8 os_prio=0 tid=0x00007ef5e000c000 nid=0x1c77 sleeping[0x00007efae7f9a000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at sun.net.www.http.KeepAliveCache.run(KeepAliveCache.java:172) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #847 daemon prio=10 os_prio=0 tid=0x00007ef3e005b000 nid=0x14d7 waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.Selector.poll(Selector.java:389) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6ed68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d32a38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe50b07e10> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d32d968> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c50eef0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at java.util.Formatter$FormatSpecifier.checkInteger(Formatter.java:2987) at java.util.Formatter$FormatSpecifier.(Formatter.java:2729) at java.util.Formatter.parse(Formatter.java:2560) at java.util.Formatter.format(Formatter.java:2501) at java.util.Formatter.format(Formatter.java:2455) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.responseDataToLogString(FetchSessionHandler.java:349) at org.apache.kafka.clients.FetchSessionHandler.handleResponse(FetchSessionHandler.java:424) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff334971e8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5ed063d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdaa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007f04ec000880> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8b59ab50> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b17ec68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1070" #621 prio=5 os_prio=0 tid=0x00007ef65e5db800 nid=0x8af7 runnable [0x00007ef7ee7fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bbad0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91497860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effad018870> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8c643558> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c930078> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:120) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed4a23428> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efede18a448> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c231a78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe780c9d88> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c022990> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9144b4c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839bff90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed93fc360> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5ddec278> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c230740> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e9ad30> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e9ad18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e96010> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04467e0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04467c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0441ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c22cd68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effa833a0c8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe80123fd8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d04a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913f9898> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88cac830> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efe97702088> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effc2549758> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5e5bdc08> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b179278> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efef0776070> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d04ecb8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7e5c98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff53574020> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7dc33a70> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912a0388> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0818e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae0818c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae07df48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3e7648> (a sun.nio.ch.Util$3) - locked <0x00007efdae3e7630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3e2928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1071" #485 prio=5 os_prio=0 tid=0x00007ef65d78f000 nid=0x8a6d runnable [0x00007ef9561fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528b30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b356ce0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d02510> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe915251f0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7a3066c0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88fbc6e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe5eba5f78> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe80042568> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c62c958> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeabf370f8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd58ca9508> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae9e0910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafad2c98> (a sun.nio.ch.Util$3) - locked <0x00007efdafad2c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafacdfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf181c70> (a sun.nio.ch.Util$3) - locked <0x00007efdaf181c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf17cf50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab8eeae0> (a sun.nio.ch.Util$3) - locked <0x00007efdab8eeac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab8e9d28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe50ef7508> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd55186028> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910b4128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff48a60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe780763f8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6bf08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c19caf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff13b1b6e0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7d4d2558> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d724590> (a sun.nio.ch.Util$3) - locked <0x00007efe7d7245a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d724518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb08d5160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5d366e00> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa17a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:43) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe807d0f60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac79d3a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:519) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb37bf788> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5078c748> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad1da880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe56085fa8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a4ec530> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae889128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.clients.FetchSessionHandler.responseDataToLogString(FetchSessionHandler.java:349) at org.apache.kafka.clients.FetchSessionHandler.handleResponse(FetchSessionHandler.java:424) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:696) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at java.util.LinkedHashMap.newNode(LinkedHashMap.java:256) at java.util.HashMap.putVal(HashMap.java:642) at java.util.HashMap.put(HashMap.java:612) at org.apache.kafka.common.requests.FetchRequest.(FetchRequest.java:399) at org.apache.kafka.common.requests.AbstractRequest.parseRequest(AbstractRequest.java:144) at org.apache.kafka.common.requests.RequestContext.parseRequest(RequestContext.java:64) at kafka.network.RequestChannel$Request.(RequestChannel.scala:89) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:696) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at java.util.regex.Pattern.matcher(Pattern.java:1093) at org.apache.kafka.common.security.kerberos.KerberosRule.replaceParameters(KerberosRule.java:120) at org.apache.kafka.common.security.kerberos.KerberosRule.apply(KerberosRule.java:180) at org.apache.kafka.common.security.kerberos.KerberosShortNamer.shortName(KerberosShortNamer.java:93) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.applyKerberosShortNamer(DefaultKafkaPrincipalBuilder.java:131) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.build(DefaultKafkaPrincipalBuilder.java:120) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.principal(SaslServerAuthenticator.java:286) at org.apache.kafka.common.network.KafkaChannel.principal(KafkaChannel.java:114) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:695) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 waiting on condition [0x00007efc05a3c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 waiting on condition [0x00007efc05b3d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:293) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 waiting on condition [0x00007efc06e5b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: RUNNABLE at kafka.server.ReplicaManager$$anonfun$24.apply(ReplicaManager.scala:842) at kafka.server.ReplicaManager$$anonfun$24.apply(ReplicaManager.scala:841) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:841) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d waiting on condition [0x00007ef603e76000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007effd30bc078> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 waiting on condition [0x00007ef5383b3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b179278> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 waiting on condition [0x00007ef530355000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f waiting on condition [0x00007ef538248000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d waiting on condition [0x00007ef600175000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a runnable [0x00007efb00172000] java.lang.Thread.State: RUNNABLE at sun.nio.fs.UnixNativeDispatcher.rename0(Native Method) at sun.nio.fs.UnixNativeDispatcher.rename(UnixNativeDispatcher.java:183) at sun.nio.fs.UnixCopyFile.move(UnixCopyFile.java:388) at sun.nio.fs.UnixFileSystemProvider.move(UnixFileSystemProvider.java:262) at java.nio.file.Files.move(Files.java:1395) at org.apache.kafka.common.utils.Utils.atomicMoveWithFallback(Utils.java:783) at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:72) at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50) - locked <0x00007efe7a346438> (a java.lang.Object) at kafka.server.checkpoints.LeaderEpochCheckpointFile.write(LeaderEpochCheckpointFile.scala:64) at kafka.server.epoch.LeaderEpochFileCache.kafka$server$epoch$LeaderEpochFileCache$$flush(LeaderEpochFileCache.scala:193) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$clearAndFlushEarliest$1.apply$mcV$sp(LeaderEpochFileCache.scala:157) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$clearAndFlushEarliest$1.apply(LeaderEpochFileCache.scala:148) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$clearAndFlushEarliest$1.apply(LeaderEpochFileCache.scala:148) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.clearAndFlushEarliest(LeaderEpochFileCache.scala:148) at kafka.log.Log$$anonfun$maybeIncrementLogStartOffset$1.apply$mcV$sp(Log.scala:945) - locked <0x00007efe7a345f10> (a java.lang.Object) at kafka.log.Log$$anonfun$maybeIncrementLogStartOffset$1.apply(Log.scala:940) at kafka.log.Log$$anonfun$maybeIncrementLogStartOffset$1.apply(Log.scala:940) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.maybeIncrementLogStartOffset(Log.scala:939) at kafka.log.Log$$anonfun$deleteSegments$1.apply$mcI$sp(Log.scala:1312) - locked <0x00007efe7a345f10> (a java.lang.Object) at kafka.log.Log$$anonfun$deleteSegments$1.apply(Log.scala:1302) at kafka.log.Log$$anonfun$deleteSegments$1.apply(Log.scala:1302) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.deleteSegments(Log.scala:1302) at kafka.log.Log.deleteOldSegments(Log.scala:1297) - locked <0x00007efe7a345f10> (a java.lang.Object) at kafka.log.Log.deleteRetentionMsBreachedSegments(Log.scala:1369) at kafka.log.Log.deleteOldSegments(Log.scala:1363) at kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:874) at kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:872) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.immutable.List.foreach(List.scala:392) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.LogManager.cleanupLogs(LogManager.scala:872) at kafka.log.LogManager$$anonfun$startup$1.apply$mcV$sp(LogManager.scala:395) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506d7910> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00007efe7a346488> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 runnable [0x00007efb141d5000] java.lang.Thread.State: RUNNABLE at java.io.FileDescriptor.sync(Native Method) at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:67) at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50) - locked <0x00007efd8311e6d0> (a java.lang.Object) at kafka.server.checkpoints.OffsetCheckpointFile.write(OffsetCheckpointFile.scala:59) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$50.apply(ReplicaManager.scala:1402) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$50.apply(ReplicaManager.scala:1402) at scala.Option.foreach(Option.scala:257) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1402) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1399) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.immutable.HashMap$HashMap1.foreach(HashMap.scala:221) at scala.collection.immutable.HashMap$HashTrieMap.foreach(HashMap.scala:428) at scala.collection.immutable.HashMap$HashTrieMap.foreach(HashMap.scala:428) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.server.ReplicaManager.checkpointHighWatermarks(ReplicaManager.scala:1399) at kafka.server.ReplicaManager$$anonfun$1.apply$mcV$sp(ReplicaManager.scala:244) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506fda08> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting for monitor entry [0x00007efb1c231000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.logSegments(Log.scala:1711) - waiting to lock <0x00007efe7a345f10> (a java.lang.Object) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506b46e0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efe7a345f10> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f runnable [0x00007efb3c1e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 waiting on condition [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheSanitizedTimelineMetric(KafkaTimelineMetricsReporter.java:475) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:418) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:13:48 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #847 daemon prio=10 os_prio=0 tid=0x00007ef3e005b000 nid=0x14d7 waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01911d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb01911b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb018c4b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a3d18> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a3d00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb049eff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff024f0d98> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4e0908> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bb2b880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf12f240> (a sun.nio.ch.Util$3) - locked <0x00007efdaf12f228> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf12a520> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1070" #621 prio=5 os_prio=0 tid=0x00007ef65e5db800 nid=0x8af7 runnable [0x00007ef7ee7fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bbad0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91012370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff270b1a38> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8bc59980> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839c8628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff4a715b30> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd50a3f708> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b36cf30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91458aa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a081348> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf2d98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efee82f95d0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd51a32948> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c22cd68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0201b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0201a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf01b498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe0fd732e0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf98c380> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a5562a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d05b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0818e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae0818c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae07df48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3e7648> (a sun.nio.ch.Util$3) - locked <0x00007efdae3e7630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3e2928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1071" #485 prio=5 os_prio=0 tid=0x00007ef65d78f000 nid=0x8a6d runnable [0x00007ef9561fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528b30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1bc5d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb49580a8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd55ad5c00> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7dc701b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5e5bc700> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff9f928> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb06e3918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae9e0910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafad2c98> (a sun.nio.ch.Util$3) - locked <0x00007efdafad2c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafacdfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effad5ce928> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe789b11f0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa4ab8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff48cd5168> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab8eeae0> (a sun.nio.ch.Util$3) - locked <0x00007efdab8eeac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab8e9d28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80bfbc98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe7d736ca0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae065670> (a sun.nio.ch.Util$3) - locked <0x00007efdae065658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae060950> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad50e370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.ConcurrentHashMap.tabAt(ConcurrentHashMap.java:755) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:938) at kafka.utils.Pool.get(Pool.scala:66) at kafka.cluster.Partition.getReplica(Partition.scala:188) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:410) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:417) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:285) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:281) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:281) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb075d8c0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac7f3120> (a sun.nio.ch.Util$3) - locked <0x00007efdac7f3108> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac7ee440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff0f5cfe68> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007efe39d94ba8> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) - locked <0x00007efd8682c8b0> (a java.lang.Object) at org.apache.kafka.common.network.Selector.wakeup(Selector.java:305) at kafka.network.Processor.wakeup(SocketServer.scala:870) at kafka.network.Processor.enqueueResponse(SocketServer.scala:829) at kafka.network.RequestChannel.sendResponse(RequestChannel.scala:336) at kafka.server.KafkaApis.kafka$server$KafkaApis$$sendResponse(KafkaApis.scala:2362) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2358) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c waiting on condition [0x00007ef5f03fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 waiting on condition [0x00007ef533f12000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c22cd68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a runnable [0x00007efb00172000] java.lang.Thread.State: RUNNABLE at sun.nio.fs.UnixNativeDispatcher.rename0(Native Method) at sun.nio.fs.UnixNativeDispatcher.rename(UnixNativeDispatcher.java:183) at sun.nio.fs.UnixCopyFile.move(UnixCopyFile.java:388) at sun.nio.fs.UnixFileSystemProvider.move(UnixFileSystemProvider.java:262) at java.nio.file.Files.move(Files.java:1395) at org.apache.kafka.common.utils.Utils.atomicMoveWithFallback(Utils.java:783) at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:72) at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50) - locked <0x00007efd519e6b30> (a java.lang.Object) at kafka.server.checkpoints.LeaderEpochCheckpointFile.write(LeaderEpochCheckpointFile.scala:64) at kafka.server.epoch.LeaderEpochFileCache.kafka$server$epoch$LeaderEpochFileCache$$flush(LeaderEpochFileCache.scala:193) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$clearAndFlushEarliest$1.apply$mcV$sp(LeaderEpochFileCache.scala:157) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$clearAndFlushEarliest$1.apply(LeaderEpochFileCache.scala:148) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$clearAndFlushEarliest$1.apply(LeaderEpochFileCache.scala:148) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.clearAndFlushEarliest(LeaderEpochFileCache.scala:148) at kafka.log.Log$$anonfun$maybeIncrementLogStartOffset$1.apply$mcV$sp(Log.scala:945) - locked <0x00007efd519b8720> (a java.lang.Object) at kafka.log.Log$$anonfun$maybeIncrementLogStartOffset$1.apply(Log.scala:940) at kafka.log.Log$$anonfun$maybeIncrementLogStartOffset$1.apply(Log.scala:940) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.maybeIncrementLogStartOffset(Log.scala:939) at kafka.log.Log$$anonfun$deleteSegments$1.apply$mcI$sp(Log.scala:1312) - locked <0x00007efd519b8720> (a java.lang.Object) at kafka.log.Log$$anonfun$deleteSegments$1.apply(Log.scala:1302) at kafka.log.Log$$anonfun$deleteSegments$1.apply(Log.scala:1302) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.deleteSegments(Log.scala:1302) at kafka.log.Log.deleteOldSegments(Log.scala:1297) - locked <0x00007efd519b8720> (a java.lang.Object) at kafka.log.Log.deleteRetentionMsBreachedSegments(Log.scala:1369) at kafka.log.Log.deleteOldSegments(Log.scala:1363) at kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:874) at kafka.log.LogManager$$anonfun$cleanupLogs$3.apply(LogManager.scala:872) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.immutable.List.foreach(List.scala:392) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.LogManager.cleanupLogs(LogManager.scala:872) at kafka.log.LogManager$$anonfun$startup$1.apply$mcV$sp(LogManager.scala:395) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506d7910> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00007efd519e6b80> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 runnable [0x00007efb141d5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 runnable [0x00007efb1d346000] java.lang.Thread.State: RUNNABLE at java.io.UnixFileSystem.list(Native Method) at java.io.File.list(File.java:1122) at java.io.File.listFiles(File.java:1207) at kafka.log.ProducerStateManager$.listSnapshotFiles(ProducerStateManager.scala:446) at kafka.log.ProducerStateManager$.kafka$log$ProducerStateManager$$deleteSnapshotFiles(ProducerStateManager.scala:456) at kafka.log.ProducerStateManager$.deleteSnapshotsBefore(ProducerStateManager.scala:453) at kafka.log.ProducerStateManager.deleteSnapshotsBefore(ProducerStateManager.scala:762) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1568) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting for monitor entry [0x00007efb3c1e2000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.logSegments(Log.scala:1711) - waiting to lock <0x00007efd519b8720> (a java.lang.Object) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506d7b50> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 waiting on condition [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50044a58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:13:59 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #847 daemon prio=10 os_prio=0 tid=0x00007ef3e005b000 nid=0x14d7 waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6ed68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff7a500e48> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a3d18> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a3d00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb049eff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe8c642970> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c50eef0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b80f918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caafc08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf12f240> (a sun.nio.ch.Util$3) - locked <0x00007efdaf12f228> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf12a520> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1070" #621 prio=5 os_prio=0 tid=0x00007ef65e5db800 nid=0x8af7 runnable [0x00007ef7ee7fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bbad0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff47bffe88> (a java.lang.Object) - locked <0x00007eff47bffe68> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effab9bed50> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a081348> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0201b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0201a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf01b498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912a0388> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb475a2a8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5eea0c28> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c635498> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c632650> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1071" #485 prio=5 os_prio=0 tid=0x00007ef65d78f000 nid=0x8a6d runnable [0x00007ef9561fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528b30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff6ac30a20> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff62130e08> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5ecbe830> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb06e3918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb07d1628> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5bf5ee50> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa4ab8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007effccd0cf78> (a java.lang.Object) - locked <0x00007effccd0cf58> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe7d85bd28> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff32929f78> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd58b7d170> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1e7d48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb5acfa20> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd51b28528> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80bfbc98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe80be4f28> (a java.lang.Object) - locked <0x00007efe80be4f08> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d724590> (a sun.nio.ch.Util$3) - locked <0x00007efe7d7245a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d724518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe8033ec38> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb08d5160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb09decf8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8cb21e08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd59fcd678> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad22cb38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeac35d4a0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe780a64e8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf5750b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac7f3120> (a sun.nio.ch.Util$3) - locked <0x00007efdac7f3108> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac7ee440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdafbc8780> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 waiting on condition [0x00007efbff4f3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd86690610> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) - locked <0x00007efd865fb058> (a java.lang.Object) at org.apache.kafka.common.network.Selector.wakeup(Selector.java:305) at kafka.network.Processor.wakeup(SocketServer.scala:870) at kafka.network.Processor.enqueueResponse(SocketServer.scala:829) at kafka.network.RequestChannel.sendResponse(RequestChannel.scala:336) at kafka.server.KafkaApis.kafka$server$KafkaApis$$sendResponse(KafkaApis.scala:2362) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2358) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 waiting on condition [0x00007ef6983fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: RUNNABLE at kafka.server.FetchSessionCache.get(FetchSession.scala:567) - locked <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.FetchManager.newContext(FetchSession.scala:760) - locked <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007efdfc01f3d0> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 waiting on condition [0x00007ef530355000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c1e7d48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.partitionCount(AbstractFetcherThread.scala:392) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:159) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:158) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.server.AbstractFetcherManager.shutdownIdleFetcherThreads(AbstractFetcherManager.scala:158) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1130) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 runnable [0x00007efb1d346000] java.lang.Thread.State: RUNNABLE at java.io.UnixFileSystem.getBooleanAttributes0(Native Method) at java.io.UnixFileSystem.getBooleanAttributes(UnixFileSystem.java:242) at java.io.File.isFile(File.java:882) at kafka.log.ProducerStateManager$$anonfun$listSnapshotFiles$1$$anonfun$apply$2.apply(ProducerStateManager.scala:447) at kafka.log.ProducerStateManager$$anonfun$listSnapshotFiles$1$$anonfun$apply$2.apply(ProducerStateManager.scala:447) at scala.collection.TraversableLike$$anonfun$filterImpl$1.apply(TraversableLike.scala:248) at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186) at scala.collection.TraversableLike$class.filterImpl(TraversableLike.scala:247) at scala.collection.TraversableLike$class.filter(TraversableLike.scala:259) at scala.collection.mutable.ArrayOps$ofRef.filter(ArrayOps.scala:186) at kafka.log.ProducerStateManager$$anonfun$listSnapshotFiles$1.apply(ProducerStateManager.scala:447) at kafka.log.ProducerStateManager$$anonfun$listSnapshotFiles$1.apply(ProducerStateManager.scala:446) at scala.Option.map(Option.scala:146) at kafka.log.ProducerStateManager$.listSnapshotFiles(ProducerStateManager.scala:446) at kafka.log.ProducerStateManager$.kafka$log$ProducerStateManager$$deleteSnapshotFiles(ProducerStateManager.scala:456) at kafka.log.ProducerStateManager$.deleteSnapshotsBefore(ProducerStateManager.scala:453) at kafka.log.ProducerStateManager.deleteSnapshotsBefore(ProducerStateManager.scala:762) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1568) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 runnable [0x00007efb2607e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 waiting on condition [0x00007efc63efd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at java.util.TreeMap.put(TreeMap.java:540) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.createTimelineMetric(KafkaTimelineMetricsReporter.java:356) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheSanitizedTimelineMetric(KafkaTimelineMetricsReporter.java:476) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:418) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:14:11 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #847 daemon prio=10 os_prio=0 tid=0x00007ef3e005b000 nid=0x14d7 waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efed0dc33e0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd559fb8b8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d32a38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff284c99e0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd58a61f70> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdcc0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf12f240> (a sun.nio.ch.Util$3) - locked <0x00007efdaf12f228> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf12a520> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1070" #621 prio=5 os_prio=0 tid=0x00007ef65e5db800 nid=0x8af7 runnable [0x00007ef7ee7fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bbad0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efeab7a9770> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe8c4fb128> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e9ad30> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e9ad18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e96010> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04467e0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04467c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0441ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:79) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd50769240> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd50c0cd70> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe8c22cd68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0201b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0201a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf01b498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe509be868> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe83542ae8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe786132b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdac560a20> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at java.util.ArrayDeque.iterator(ArrayDeque.java:611) at org.apache.kafka.clients.InFlightRequests.hasExpiredRequest(InFlightRequests.java:166) at org.apache.kafka.clients.InFlightRequests.nodesWithTimedOutRequests(InFlightRequests.java:185) at org.apache.kafka.clients.NetworkClient.handleTimedOutRequests(NetworkClient.java:694) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:523) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe789a7a08> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d8bad8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff9b196a68> (a java.lang.Object) - locked <0x00007eff9b196a48> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0818e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae0818c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae07df48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3e7648> (a sun.nio.ch.Util$3) - locked <0x00007efdae3e7630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3e2928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1071" #485 prio=5 os_prio=0 tid=0x00007ef65d78f000 nid=0x8a6d runnable [0x00007ef9561fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528b30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 waiting for monitor entry [0x00007ef993f55000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4c70> (a org.apache.log4j.Logger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf95cdc0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:519) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb3975d50> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7805f5c0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88fbc6e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at scala.collection.mutable.Buffer$.newBuilder(Buffer.scala:45) at scala.collection.generic.GenericTraversableTemplate$class.genericBuilder(GenericTraversableTemplate.scala:70) at scala.collection.AbstractTraversable.genericBuilder(Traversable.scala:104) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:57) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:52) at scala.collection.TraversableLike$class.builder$1(TraversableLike.scala:229) at scala.collection.TraversableLike$class.map(TraversableLike.scala:233) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$states(AbstractFetcherThread.scala:100) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafb04e18> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff7dee34f8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe780ccbd8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf0c2788> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51) at sun.nio.ch.IOUtil.write(IOUtil.java:148) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:504) - locked <0x00007efdb0ab1cb0> (a java.lang.Object) at java.nio.channels.SocketChannel.write(SocketChannel.java:502) at org.apache.kafka.common.network.PlaintextTransportLayer.write(PlaintextTransportLayer.java:153) at org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:60) at org.apache.kafka.common.network.KafkaChannel.send(KafkaChannel.java:339) at org.apache.kafka.common.network.KafkaChannel.write(KafkaChannel.java:310) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:512) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafad2c98> (a sun.nio.ch.Util$3) - locked <0x00007efdafad2c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafacdfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaf1b3a78> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf181c70> (a sun.nio.ch.Util$3) - locked <0x00007efdaf181c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf17cf50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab8eeae0> (a sun.nio.ch.Util$3) - locked <0x00007efdab8eeac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab8e9d28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at java.io.FileOutputStream.writeBytes(Native Method) at java.io.FileOutputStream.write(FileOutputStream.java:326) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295) at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) - locked <0x00007effaab370e0> (a java.io.OutputStreamWriter) at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59) at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324) at org.apache.log4j.DailyRollingFileAppender.subAppend(DailyRollingFileAppender.java:369) at org.apache.log4j.WriterAppender.append(WriterAppender.java:162) at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251) - locked <0x00007efd50146c78> (a org.apache.log4j.DailyRollingFileAppender) at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66) at org.apache.log4j.Category.callAppenders(Category.java:206) - locked <0x00007efd500d4c70> (a org.apache.log4j.Logger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb00ab6a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efefbd7ca10> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a05b9f8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff48a60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe4d48f348> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:519) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:120) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007f04ecca5918> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8033ec38> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb08d5160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaec6c9d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 waiting for monitor entry [0x00007efb882fd000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4c70> (a org.apache.log4j.Logger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf998560> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48) at scala.collection.TraversableOnce$class.copyToBuffer(TraversableOnce.scala:275) at scala.collection.AbstractTraversable.copyToBuffer(Traversable.scala:104) at scala.collection.MapLike$class.toBuffer(MapLike.scala:326) at scala.collection.AbstractMap.toBuffer(Map.scala:59) at scala.collection.MapLike$class.toSeq(MapLike.scala:323) at scala.collection.AbstractMap.toSeq(Map.scala:59) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:245) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac7f3120> (a sun.nio.ch.Util$3) - locked <0x00007efdac7f3108> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac7ee440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:519) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd8656d208> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 waiting on condition [0x00007efc05b3d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 waiting on condition [0x00007efc05c3e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 waiting on condition [0x00007efc05e40000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 waiting on condition [0x00007efc60c84000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaff48a60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e waiting on condition [0x00007ef5f8dfe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 runnable [0x00007efb1c231000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.unpark(Native Method) at java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:141) at java.util.concurrent.locks.AbstractQueuedSynchronizer.unparkSuccessor(AbstractQueuedSynchronizer.java:662) at java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1264) at java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:457) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1104) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efe7805f5c0> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 runnable [0x00007efb244b4000] java.lang.Thread.State: RUNNABLE at java.nio.MappedByteBuffer.force0(Native Method) at java.nio.MappedByteBuffer.force(MappedByteBuffer.java:203) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.AbstractIndex.flush(AbstractIndex.scala:155) at kafka.log.LogSegment$$anonfun$flush$1.apply$mcV$sp(LogSegment.scala:467) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) at kafka.log.LogSegment.flush(LogSegment.scala:464) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1537) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1536) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd508677c8> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00007efed6b6ac20> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.unpark(Native Method) at java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:141) at java.util.concurrent.locks.AbstractQueuedSynchronizer.unparkSuccessor(AbstractQueuedSynchronizer.java:662) at java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1264) at java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:457) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1104) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50116b30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1076) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricHolder.put(TimelineMetricsCache.java:149) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache.putTimelineMetric(TimelineMetricsCache.java:194) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheSanitizedTimelineMetric(KafkaTimelineMetricsReporter.java:480) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheKafkaMetered(KafkaTimelineMetricsReporter.java:427) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processMeter(KafkaTimelineMetricsReporter.java:365) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processMeter(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Meter.processWith(Meter.java:131) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:14:23 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #847 daemon prio=10 os_prio=0 tid=0x00007ef3e005b000 nid=0x14d7 waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efefd66a0b8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8b0005e8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c20a718> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:519) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff87ad2f00> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd59fe4d00> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6ed68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a55ade8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4f3790> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d32a38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe5870f640> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efee3c865b0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c50eef0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5baca408> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9311a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdcc0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff6e4dda68> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe789dbe18> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caafc08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e90308> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4e0908> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at java.util.ArrayDeque.iterator(ArrayDeque.java:611) at org.apache.kafka.clients.InFlightRequests.hasExpiredRequest(InFlightRequests.java:166) at org.apache.kafka.clients.InFlightRequests.nodesWithTimedOutRequests(InFlightRequests.java:185) at org.apache.kafka.clients.NetworkClient.handleTimedOutRequests(NetworkClient.java:694) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:523) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5066a260> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b17ec68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.clients.InFlightRequests.nodesWithTimedOutRequests(InFlightRequests.java:181) at org.apache.kafka.clients.NetworkClient.handleTimedOutRequests(NetworkClient.java:694) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:523) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7d752040> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839bff90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff5426dbc8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d004410> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913427c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7a081348> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf2d98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at java.util.HashMap$EntrySet.iterator(HashMap.java:1014) at org.apache.kafka.clients.NetworkClient.handleDisconnections(NetworkClient.java:802) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:520) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c22cd68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0201b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0201a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf01b498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913f9898> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d8bad8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7869daf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff14a99b00> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8075cdc8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b179278> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7e5c98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff50188fd8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5ee7e580> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf98c380> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c638680> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:79) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd59a37360> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd59a3b878> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe912a0388> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c635498> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.Selector.poll(Selector.java:389) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1071" #485 prio=5 os_prio=0 tid=0x00007ef65d78f000 nid=0x8a6d runnable [0x00007ef9561fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528b30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe6ebd0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff064f2520> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5e1cec78> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80da6288> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7805b268> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1bc5d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe62a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88fbc6e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effd3bbbea0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d0895b8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf0c2788> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe916a53c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff335ba098> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a31d9f8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7a080888> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaf1b3a78> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa4ab8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c17fa60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7a34c840> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5a75fd40> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb00ab6a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe5e762158> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe80334ba8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910b4128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff13b1b700> (a java.lang.Object) - locked <0x00007eff13b1b6e0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0945ec0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7a3a7ef0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0f4d810> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at java.util.ArrayDeque.iterator(ArrayDeque.java:611) at org.apache.kafka.clients.InFlightRequests.hasExpiredRequest(InFlightRequests.java:166) at org.apache.kafka.clients.InFlightRequests.nodesWithTimedOutRequests(InFlightRequests.java:185) at org.apache.kafka.clients.NetworkClient.handleTimedOutRequests(NetworkClient.java:694) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:523) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeb51428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efef08ab660> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd506fd370> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae7e7f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdae47a5f8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb09decf8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad50e370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad7cb4e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at java.util.HashMap$EntrySet.iterator(HashMap.java:1014) at org.apache.kafka.common.network.Selector.clear(Selector.java:661) at org.apache.kafka.common.network.Selector.poll(Selector.java:389) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effd3004860> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efede69fc00> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e80910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad1da880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 waiting on condition [0x00007efbff5f4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 waiting on condition [0x00007efbff6f5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd867076d8> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 waiting on condition [0x00007efc04513000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 waiting on condition [0x00007efc0593b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 waiting on condition [0x00007efc05c3e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd832e97e0> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b waiting on condition [0x00007efc06c59000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8caafc08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: RUNNABLE at kafka.server.FetchSessionCache.get(FetchSession.scala:567) - locked <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.FetchManager.newContext(FetchSession.scala:760) - locked <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 waiting on condition [0x00007efc60c84000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c waiting on condition [0x00007ef5f03fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 runnable [0x00007efb1d346000] java.lang.Thread.State: RUNNABLE at java.io.UnixFileSystem.list(Native Method) at java.io.File.list(File.java:1122) at java.io.File.listFiles(File.java:1207) at kafka.log.ProducerStateManager$.listSnapshotFiles(ProducerStateManager.scala:446) at kafka.log.ProducerStateManager$.kafka$log$ProducerStateManager$$deleteSnapshotFiles(ProducerStateManager.scala:456) at kafka.log.ProducerStateManager$.deleteSnapshotsBefore(ProducerStateManager.scala:453) at kafka.log.ProducerStateManager.deleteSnapshotsBefore(ProducerStateManager.scala:762) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1568) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 runnable [0x00007efb244b4000] java.lang.Thread.State: RUNNABLE at java.io.FileDescriptor.sync(Native Method) at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:67) at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50) - locked <0x00007efd8311d7e8> (a java.lang.Object) at kafka.server.checkpoints.OffsetCheckpointFile.write(OffsetCheckpointFile.scala:59) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$50.apply(ReplicaManager.scala:1402) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$50.apply(ReplicaManager.scala:1402) at scala.Option.foreach(Option.scala:257) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1402) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1399) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.immutable.HashMap$HashMap1.foreach(HashMap.scala:221) at scala.collection.immutable.HashMap$HashTrieMap.foreach(HashMap.scala:428) at scala.collection.immutable.HashMap$HashTrieMap.foreach(HashMap.scala:428) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.server.ReplicaManager.checkpointHighWatermarks(ReplicaManager.scala:1399) at kafka.server.ReplicaManager$$anonfun$1.apply$mcV$sp(ReplicaManager.scala:244) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd508677c8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 runnable [0x00007efb2607e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f runnable [0x00007efb3c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.fs.UnixNativeDispatcher.unlink0(Native Method) at sun.nio.fs.UnixNativeDispatcher.unlink(UnixNativeDispatcher.java:141) at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:231) at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108) at java.nio.file.Files.deleteIfExists(Files.java:1165) at org.apache.kafka.common.record.FileRecords.deleteIfExists(FileRecords.java:187) at kafka.log.LogSegment$$anonfun$deleteIfExists$1$$anonfun$apply$mcV$sp$1.apply$mcZ$sp(LogSegment.scala:598) at kafka.log.LogSegment.kafka$log$LogSegment$$delete$1(LogSegment.scala:587) at kafka.log.LogSegment$$anonfun$deleteIfExists$1.apply$mcV$sp(LogSegment.scala:598) at kafka.utils.CoreUtils$$anonfun$tryAll$1.apply(CoreUtils.scala:115) at kafka.utils.CoreUtils$$anonfun$tryAll$1.apply(CoreUtils.scala:114) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.utils.CoreUtils$.tryAll(CoreUtils.scala:114) at kafka.log.LogSegment.deleteIfExists(LogSegment.scala:597) at kafka.log.Log$$anonfun$kafka$log$Log$$deleteSeg$1$1.apply$mcV$sp(Log.scala:1757) at kafka.log.Log$$anonfun$kafka$log$Log$$deleteSeg$1$1.apply(Log.scala:1757) at kafka.log.Log$$anonfun$kafka$log$Log$$deleteSeg$1$1.apply(Log.scala:1757) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.kafka$log$Log$$deleteSeg$1(Log.scala:1756) at kafka.log.Log$$anonfun$kafka$log$Log$$asyncDeleteSegment$1.apply$mcV$sp(Log.scala:1760) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506d7b50> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd5004e058> (a java.util.concurrent.ThreadPoolExecutor$Worker) "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.ConcurrentSkipListMap.doGet(ConcurrentSkipListMap.java:802) at java.util.concurrent.ConcurrentSkipListMap.get(ConcurrentSkipListMap.java:1546) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricHolder.put(TimelineMetricsCache.java:152) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache.putTimelineMetric(TimelineMetricsCache.java:194) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheSanitizedTimelineMetric(KafkaTimelineMetricsReporter.java:480) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:418) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 306 ========== 2020-12-28 13:14:35 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "Keep-Alive-Timer" #927 daemon prio=8 os_prio=0 tid=0x00007ef5e000c000 nid=0x1f0a waiting on condition [0x00007ef7ee7fc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at sun.net.www.http.KeepAliveCache.run(KeepAliveCache.java:172) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #847 daemon prio=10 os_prio=0 tid=0x00007ef3e005b000 nid=0x14d7 waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff9b1b3918> (a java.lang.Object) - locked <0x00007eff9b1b38f8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01911d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb01911b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb018c4b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdb049f578> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80cfbaf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache.validateAndMaybeWarn(LeaderEpochFileCache.scala:200) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:66) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd50843330> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd50ea3bc8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe8b940d58> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff54e4db48> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7802b708> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e90308> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe8c2fe460> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdaa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf12f240> (a sun.nio.ch.Util$3) - locked <0x00007efdaf12f228> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf12a520> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efef3008b30> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5e3b6300> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac6e90b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff270b1a38> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8bc59980> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839c8628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c231a78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04467e0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04467c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0441ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d04a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b927ba8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913f9898> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe0212dd18> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5078e1d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88cac830> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff872269e0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efee3cb11d8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c638680> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0818e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae0818c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae07df48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe6ebd0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b356ce0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb70c7928> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8b6f4da8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80da6288> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff9f928> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff2896fbd0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff3358b0f0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a080888> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff21a690d0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5540f8d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab8eeae0> (a sun.nio.ch.Util$3) - locked <0x00007efdab8eeac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab8e9d28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effae9d28a8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a053fd8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910b4128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efdfc1c9408> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe780763f8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5dfa52c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6bf08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c19caf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efee3d031e8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1e7d48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efede42d918> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf732808> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff6461b2e0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd50745198> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0945ec0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeb51428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d724590> (a sun.nio.ch.Util$3) - locked <0x00007efe7d7245a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d724518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe835414f8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae7e7f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff45a29648> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd55b2d678> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaec6c9d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeed2b6078> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd50eda2d8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8cb21e08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe8d8c1dc0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5ea86e88> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad22cb38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e80910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe80330140> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac79d3a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) - <0x00007efe80330758> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff53af6ca8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78981838> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac644d50> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efee3d085a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae889128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.ConcurrentHashMap.tabAt(ConcurrentHashMap.java:755) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:938) at kafka.utils.Pool.get(Pool.scala:66) at kafka.cluster.Partition.getReplica(Partition.scala:188) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:410) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:417) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:285) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:281) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:281) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaea48250> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeef93e1e8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78a4af70> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae24f450> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaee430d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff8cd227f0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at java.util.Arrays.copyOfRange(Arrays.java:3664) at java.lang.String.(String.java:207) at java.lang.String.substring(String.java:1969) at java.lang.String.subSequence(String.java:2003) at java.util.regex.Matcher.getSubSequence(Matcher.java:1294) at java.util.regex.Matcher.group(Matcher.java:541) at org.apache.kafka.common.security.kerberos.KerberosName.parse(KerberosName.java:59) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.applyKerberosShortNamer(DefaultKafkaPrincipalBuilder.java:129) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.build(DefaultKafkaPrincipalBuilder.java:120) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.principal(SaslServerAuthenticator.java:286) at org.apache.kafka.common.network.KafkaChannel.principal(KafkaChannel.java:114) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:695) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 waiting on condition [0x00007efbffffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at java.util.regex.Pattern.matcher(Pattern.java:1093) at org.apache.kafka.common.security.kerberos.KerberosRule.apply(KerberosRule.java:181) at org.apache.kafka.common.security.kerberos.KerberosShortNamer.shortName(KerberosShortNamer.java:93) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.applyKerberosShortNamer(DefaultKafkaPrincipalBuilder.java:131) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.build(DefaultKafkaPrincipalBuilder.java:120) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.principal(SaslServerAuthenticator.java:286) at org.apache.kafka.common.network.KafkaChannel.principal(KafkaChannel.java:114) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:695) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 waiting on condition [0x00007efc05a3c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 waiting on condition [0x00007efc05c3e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51) at sun.nio.ch.IOUtil.write(IOUtil.java:148) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:504) - locked <0x00007eff8cd4ccf0> (a java.lang.Object) at java.nio.channels.SocketChannel.write(SocketChannel.java:502) at org.apache.kafka.common.network.PlaintextTransportLayer.write(PlaintextTransportLayer.java:153) at org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:60) at org.apache.kafka.common.record.MultiRecordsSend.writeTo(MultiRecordsSend.java:93) at org.apache.kafka.common.network.KafkaChannel.send(KafkaChannel.java:339) at org.apache.kafka.common.network.KafkaChannel.write(KafkaChannel.java:310) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:512) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f waiting on condition [0x00007efc05f41000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.protocol.types.Schema.sizeOf(Schema.java:92) at org.apache.kafka.common.protocol.types.ArrayOf.sizeOf(ArrayOf.java:90) at org.apache.kafka.common.protocol.types.Schema.sizeOf(Schema.java:92) at org.apache.kafka.common.protocol.types.ArrayOf.sizeOf(ArrayOf.java:90) at org.apache.kafka.common.protocol.types.Schema.sizeOf(Schema.java:92) at org.apache.kafka.common.protocol.types.Struct.sizeOf(Struct.java:357) at org.apache.kafka.common.requests.FetchResponse.sizeOf(FetchResponse.java:536) at kafka.server.IncrementalFetchContext.getResponseSize(FetchSession.scala:457) - locked <0x00007efdfc01f3d0> (a kafka.server.FetchSession) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:648) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 waiting on condition [0x00007efc06e5b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 waiting on condition [0x00007efc09208000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe839c8628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 waiting on condition [0x00007efc60c84000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 waiting on condition [0x00007ef532e13000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 waiting on condition [0x00007efaf114e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 waiting on condition [0x00007efbe5bd9000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efd5540f8d0> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 runnable [0x00007efb244b4000] java.lang.Thread.State: RUNNABLE at sun.nio.fs.UnixNativeDispatcher.unlink0(Native Method) at sun.nio.fs.UnixNativeDispatcher.unlink(UnixNativeDispatcher.java:141) at sun.nio.fs.UnixFileSystemProvider.implDelete(UnixFileSystemProvider.java:231) at sun.nio.fs.AbstractFileSystemProvider.deleteIfExists(AbstractFileSystemProvider.java:108) at java.nio.file.Files.deleteIfExists(Files.java:1165) at org.apache.kafka.common.record.FileRecords.deleteIfExists(FileRecords.java:187) at kafka.log.LogSegment$$anonfun$deleteIfExists$1$$anonfun$apply$mcV$sp$1.apply$mcZ$sp(LogSegment.scala:598) at kafka.log.LogSegment.kafka$log$LogSegment$$delete$1(LogSegment.scala:587) at kafka.log.LogSegment$$anonfun$deleteIfExists$1.apply$mcV$sp(LogSegment.scala:598) at kafka.utils.CoreUtils$$anonfun$tryAll$1.apply(CoreUtils.scala:115) at kafka.utils.CoreUtils$$anonfun$tryAll$1.apply(CoreUtils.scala:114) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.utils.CoreUtils$.tryAll(CoreUtils.scala:114) at kafka.log.LogSegment.deleteIfExists(LogSegment.scala:597) at kafka.log.Log$$anonfun$kafka$log$Log$$deleteSeg$1$1.apply$mcV$sp(Log.scala:1757) at kafka.log.Log$$anonfun$kafka$log$Log$$deleteSeg$1$1.apply(Log.scala:1757) at kafka.log.Log$$anonfun$kafka$log$Log$$deleteSeg$1$1.apply(Log.scala:1757) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.kafka$log$Log$$deleteSeg$1(Log.scala:1756) at kafka.log.Log$$anonfun$kafka$log$Log$$asyncDeleteSegment$1.apply$mcV$sp(Log.scala:1760) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd508677c8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 runnable [0x00007efb246ae000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 waiting on condition [0x00007efc63efd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at kafka.server.ReplicaManager$$anon$5$$anonfun$value$1.apply(ReplicaManager.scala:232) at scala.collection.TraversableOnce$$anonfun$count$1.apply(TraversableOnce.scala:116) at scala.collection.TraversableOnce$$anonfun$count$1.apply(TraversableOnce.scala:115) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.TraversableOnce$class.count(TraversableOnce.scala:115) at scala.collection.AbstractIterator.count(Iterator.scala:1334) at kafka.server.ReplicaManager$$anon$5.value(ReplicaManager.scala:232) at kafka.server.ReplicaManager$$anon$5.value(ReplicaManager.scala:231) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:418) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:14:46 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01911d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb01911b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb018c4b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a3d18> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a3d00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb049eff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c50eef0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff3c199950> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5d06ef20> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdcc0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caafc08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff4c93ac20> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4e0908> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdaa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf12f240> (a sun.nio.ch.Util$3) - locked <0x00007efdaf12f228> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf12a520> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9725a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: RUNNABLE at java.lang.String.toCharArray(String.java:2899) at java.util.Formatter$Flags.parse(Formatter.java:4438) at java.util.Formatter$FormatSpecifier.flags(Formatter.java:2640) at java.util.Formatter$FormatSpecifier.(Formatter.java:2709) at java.util.Formatter.parse(Formatter.java:2560) at java.util.Formatter.format(Formatter.java:2501) at java.util.Formatter.format(Formatter.java:2455) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.responseDataToLogString(FetchSessionHandler.java:352) at org.apache.kafka.clients.FetchSessionHandler.handleResponse(FetchSessionHandler.java:405) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac6e90b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: RUNNABLE at java.util.HashMap$KeySet.iterator(HashMap.java:917) at java.util.HashSet.iterator(HashSet.java:173) at sun.nio.ch.Util$3.iterator(Util.java:324) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:457) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effbca56b80> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe98d28748> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839c8628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff2eb574f8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9144b4c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe7d52eac0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad2637c8> (a sun.nio.ch.Util$3) - locked <0x00007efdad2637b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad25ea10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf2d98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff035d60f0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2c0980> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0ba0cf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0201b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0201a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf01b498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff3379a7b0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78076f60> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe786132b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe78559998> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913f9898> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c229448> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd55645e98> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7869daf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe919959a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff52b1ce00> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b923c78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effa2e58830> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a5562a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d05b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c635498> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3e7648> (a sun.nio.ch.Util$3) - locked <0x00007efdae3e7630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3e2928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff6ac30a40> (a java.lang.Object) - locked <0x00007eff6ac30a20> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe73edc200> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a500d60> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b356ce0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff8cf99998> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d373648> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80da6288> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a765b10> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1bc5d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efdfc8c24c0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a3ea640> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe915251f0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7dc701b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe916a53c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafad2c98> (a sun.nio.ch.Util$3) - locked <0x00007efdafad2c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafacdfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff27c5f980> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7aa23ae8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0054380> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5540f8d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe5ea2e8b0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd50ebe678> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c17fa60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe8c2dd8e0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6bf08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effa82a4ff0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe800dfdb0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0f4d810> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd59e99790> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdae47a5f8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7ac9eae0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad9988d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae065670> (a sun.nio.ch.Util$3) - locked <0x00007efdae065658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae060950> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb09decf8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeed2b6078> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd50eda2d8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8cb21e08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad50e370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff3380d1f8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d077358> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb075d8c0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe4c50aa30> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effd3004860> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efede69fc00> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e80910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe50eb7d70> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d06f330> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac79d3a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb1083340> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78a55de0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c86e598> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac644d50> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad1da880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae889128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f waiting on condition [0x00007efbff9f8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a waiting on condition [0x00007efc06d5a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007efe39d94c18> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: RUNNABLE at kafka.server.FetchSessionCache.get(FetchSession.scala:567) - locked <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.FetchManager.newContext(FetchSession.scala:760) - locked <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e waiting on condition [0x00007ef5f8dfe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe786132b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.partitionCount(AbstractFetcherThread.scala:392) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:159) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:158) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.server.AbstractFetcherManager.shutdownIdleFetcherThreads(AbstractFetcherManager.scala:158) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1130) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: RUNNABLE at scala.collection.IndexedSeqLike$class.iterator(IndexedSeqLike.scala:90) at scala.collection.mutable.ArrayBuffer.iterator(ArrayBuffer.scala:48) at scala.collection.convert.Wrappers$IterableWrapperTrait$class.iterator(Wrappers.scala:25) at scala.collection.convert.Wrappers$SeqWrapper.iterator(Wrappers.scala:65) at scala.collection.convert.Wrappers$SeqWrapper.iterator(Wrappers.scala:65) at org.apache.kafka.common.requests.MetadataResponse.toStruct(MetadataResponse.java:558) at org.apache.kafka.common.requests.AbstractResponse.serialize(AbstractResponse.java:41) at org.apache.kafka.common.requests.AbstractResponse.toSend(AbstractResponse.java:34) at org.apache.kafka.common.requests.RequestContext.buildResponse(RequestContext.java:78) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2350) at kafka.server.KafkaApis.sendResponseMaybeThrottle(KafkaApis.scala:2300) at kafka.server.KafkaApis.handleTopicMetadataRequest(KafkaApis.scala:1092) at kafka.server.KafkaApis.handle(KafkaApis.scala:109) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 waiting on condition [0x00007ef7d5223000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d waiting on condition [0x00007ef600175000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 runnable [0x00007efb0058e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efe98d28748> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 waiting on condition [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 waiting on condition [0x00007efcb8341000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50044a58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:14:58 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaf8070c0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd59eedea0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe78c00498> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91358138> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6ed68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a3d18> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a3d00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb049eff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c50eef0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9311a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4c4268> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caafc08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e90308> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe8374a0b8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bb2b880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff5336ac98> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2f6500> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b17ec68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effc4b41420> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8cc4c808> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b3a0648> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7f6b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839c8628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c231a78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:207) - locked <0x00007effb5095908> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5078c0b8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839bff90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c230740> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e9ad30> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e9ad18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e96010> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf2d98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0ba0cf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efed0d24e80> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6400> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.ConcurrentHashMap.tabAt(ConcurrentHashMap.java:755) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:938) at kafka.utils.Pool.get(Pool.scala:66) at kafka.cluster.Partition.getReplica(Partition.scala:188) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:410) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:417) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:285) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:281) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:281) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c229448> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd55cdcdf0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d8bad8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaf03b7d8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7e5c98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdafd48e80> (a java.lang.Object) - locked <0x00007efdafd48e60> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b923c78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff4d2f1ad0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efed0f65988> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c638680> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d05b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912a0388> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5a114450> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c635498> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c632650> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c5135c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.internals.PartitionStates.partitionStates(PartitionStates.java:82) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$states(AbstractFetcherThread.scala:100) at kafka.server.AbstractFetcherThread$$anonfun$3.apply(AbstractFetcherThread.scala:127) at kafka.server.AbstractFetcherThread$$anonfun$3.apply(AbstractFetcherThread.scala:127) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.maybeTruncate(AbstractFetcherThread.scala:127) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf95cdc0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1bc5d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe805beed8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe915251f0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efd88cb9ad8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5df79378> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c62c958> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe8d08bd98> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efed0f765a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff9f928> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe916a53c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0054380> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb1a5b780> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2b3520> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd50744af0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c19caf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1e7d48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80bfbc98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0945ec0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9738a428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5de95c98> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeb51428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeadc26288> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efee3ccfac0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7d4d2558> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd59e99790> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb08d5160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb09decf8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff03f0a040> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd55b2d678> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaec6c9d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91696b48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5ac58800> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa17a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8cb21e08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7a30f0b0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c230c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e80910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.ConcurrentHashMap.tabAt(ConcurrentHashMap.java:755) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:938) at kafka.utils.Pool.get(Pool.scala:66) at kafka.cluster.Partition.getReplica(Partition.scala:188) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:410) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:417) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:285) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:281) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:281) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac79d3a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff0f5cfe68> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad1da880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007f001488cde8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efeadfc6228> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae24f450> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1c1118> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5bb2b5c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaee430d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 waiting on condition [0x00007efbff4f3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff63c11c90> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f waiting on condition [0x00007efbff9f8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d waiting on condition [0x00007efbffbfa000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51) at sun.nio.ch.IOUtil.write(IOUtil.java:148) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:504) - locked <0x00007eff061664d0> (a java.lang.Object) at java.nio.channels.SocketChannel.write(SocketChannel.java:502) at org.apache.kafka.common.network.PlaintextTransportLayer.write(PlaintextTransportLayer.java:153) at org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:60) at org.apache.kafka.common.record.MultiRecordsSend.writeTo(MultiRecordsSend.java:93) at org.apache.kafka.common.network.KafkaChannel.send(KafkaChannel.java:339) at org.apache.kafka.common.network.KafkaChannel.write(KafkaChannel.java:310) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:512) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:501) - locked <0x00007eff639614b0> (a java.lang.Object) - locked <0x00007eff639614a0> (a java.lang.Object) at java.nio.channels.SocketChannel.write(SocketChannel.java:502) at org.apache.kafka.common.network.PlaintextTransportLayer.write(PlaintextTransportLayer.java:153) at org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:60) at org.apache.kafka.common.record.MultiRecordsSend.writeTo(MultiRecordsSend.java:93) at org.apache.kafka.common.network.KafkaChannel.send(KafkaChannel.java:339) at org.apache.kafka.common.network.KafkaChannel.write(KafkaChannel.java:310) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:512) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 waiting on condition [0x00007efc05b3d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 waiting on condition [0x00007efc05c3e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 waiting on condition [0x00007efc05d3f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c waiting on condition [0x00007efc06542000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.protocol.types.Struct.(Struct.java:38) at org.apache.kafka.common.protocol.types.Struct.instance(Struct.java:326) at org.apache.kafka.common.protocol.types.Struct.instance(Struct.java:343) at org.apache.kafka.common.requests.FetchResponse.toStruct(FetchResponse.java:490) at org.apache.kafka.common.requests.FetchResponse.toStruct(FetchResponse.java:359) at org.apache.kafka.common.requests.FetchResponse.toSend(FetchResponse.java:365) at org.apache.kafka.common.requests.RequestContext.buildResponse(RequestContext.java:78) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2350) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 waiting on condition [0x00007efc09208000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 waiting on condition [0x00007efc60c84000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007eff35e5b1b0> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 waiting on condition [0x00007ef5383b3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 waiting on condition [0x00007efaf114e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe839bff90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 runnable [0x00007efb0058e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 runnable [0x00007efb1c231000] java.lang.Thread.State: RUNNABLE at java.nio.MappedByteBuffer.force0(Native Method) at java.nio.MappedByteBuffer.force(MappedByteBuffer.java:203) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.AbstractIndex.flush(AbstractIndex.scala:155) at kafka.log.LogSegment$$anonfun$flush$1.apply$mcV$sp(LogSegment.scala:467) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) at kafka.log.LogSegment.flush(LogSegment.scala:464) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1537) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1536) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506b46e0> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00007eff62acf2b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efd5bb2b5c8> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 runnable [0x00007efb246ae000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.force0(Native Method) at sun.nio.ch.FileDispatcherImpl.force(FileDispatcherImpl.java:80) at sun.nio.ch.FileChannelImpl.force(FileChannelImpl.java:388) at org.apache.kafka.common.record.FileRecords.flush(FileRecords.java:160) at kafka.log.LogSegment$$anonfun$flush$1.apply$mcV$sp(LogSegment.scala:465) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) at kafka.log.LogSegment.flush(LogSegment.scala:464) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1537) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1536) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50843828> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 waiting on condition [0x00007efc63efd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50044a58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:15:10 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c932810> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb90faab8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8cc36080> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad3560a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad356090> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad3512f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at java.util.regex.Matcher.(Matcher.java:225) at java.util.regex.Pattern.matcher(Pattern.java:1093) at java.util.Formatter.parse(Formatter.java:2547) at java.util.Formatter.format(Formatter.java:2501) at java.util.Formatter.format(Formatter.java:2455) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.responseDataToLogString(FetchSessionHandler.java:349) at org.apache.kafka.clients.FetchSessionHandler.handleResponse(FetchSessionHandler.java:424) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a3d18> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a3d00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb049eff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:43) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80cfbaf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91470520> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdcc0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efff16d14f8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5acf67a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c18e558> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf12f240> (a sun.nio.ch.Util$3) - locked <0x00007efdaf12f228> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf12a520> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe8b740ce0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7412d8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe91497860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7f6b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9725a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff5b50fce8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2be8f0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b36cf30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff771e2138> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5078db08> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91458aa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e9ad30> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e9ad18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e96010> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff41613950> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a5413d8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf2d98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effadb76558> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a3cf888> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d04a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.NativeThreadSet.add(NativeThreadSet.java:46) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:207) - locked <0x00007eff8cd54df8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a327220> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b927ba8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a759398> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe786132b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913f9898> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88cac830> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efead0d1168> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd50a40010> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b179278> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7e5c98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.requests.FetchRequest$Builder.forReplica(FetchRequest.java:279) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:299) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d21f40> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d05b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache.validateAndMaybeWarn(LeaderEpochFileCache.scala:200) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:66) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7809a960> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7809b778> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe912a0388> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0818e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae0818c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae07df48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3e7648> (a sun.nio.ch.Util$3) - locked <0x00007efdae3e7630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3e2928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88fbc6e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.ConcurrentHashMap.tabAt(ConcurrentHashMap.java:755) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:938) at kafka.utils.Pool.get(Pool.scala:66) at kafka.cluster.Partition.getReplica(Partition.scala:188) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:410) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:417) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:285) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:281) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:281) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c05eb48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7dc701b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf0c2788> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe916a53c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafad2c98> (a sun.nio.ch.Util$3) - locked <0x00007efdafad2c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafacdfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf181c70> (a sun.nio.ch.Util$3) - locked <0x00007efdaf181c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf17cf50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab8eeae0> (a sun.nio.ch.Util$3) - locked <0x00007efdab8eeac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab8e9d28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe50a22378> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe832a2690> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb00ab6a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.requests.FetchRequest.toStruct(FetchRequest.java:479) at org.apache.kafka.common.requests.AbstractRequest.serialize(AbstractRequest.java:98) at org.apache.kafka.common.requests.AbstractRequest.toSend(AbstractRequest.java:91) at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:476) at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:450) at org.apache.kafka.clients.NetworkClient.send(NetworkClient.java:411) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:91) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80bfbc98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff45297b98> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at java.io.FileOutputStream.writeBytes(Native Method) at java.io.FileOutputStream.write(FileOutputStream.java:326) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295) at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) - locked <0x00007effaab370e0> (a java.io.OutputStreamWriter) at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59) at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324) at org.apache.log4j.DailyRollingFileAppender.subAppend(DailyRollingFileAppender.java:369) at org.apache.log4j.WriterAppender.append(WriterAppender.java:162) at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251) - locked <0x00007efd50146c78> (a org.apache.log4j.DailyRollingFileAppender) at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66) at org.apache.log4j.Category.callAppenders(Category.java:206) - locked <0x00007efd500d4c70> (a org.apache.log4j.Logger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7d4d2558> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe4dc41158> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d724590> (a sun.nio.ch.Util$3) - locked <0x00007efe7d7245a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d724518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae7e7f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb08d5160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe91127d70> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad7cb4e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c230c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a9a360> (a sun.nio.ch.Util$3) - locked <0x00007efda9a9a348> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a9a2d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.requests.FetchRequest$TopicAndPartitionData.batchByTopic(FetchRequest.java:248) at org.apache.kafka.common.requests.FetchRequest.toStruct(FetchRequest.java:467) at org.apache.kafka.common.requests.AbstractRequest.serialize(AbstractRequest.java:98) at org.apache.kafka.common.requests.AbstractRequest.toSend(AbstractRequest.java:91) at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:476) at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:450) at org.apache.kafka.clients.NetworkClient.send(NetworkClient.java:411) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:91) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:253) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache.validateAndMaybeWarn(LeaderEpochFileCache.scala:200) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:66) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efede1bf458> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac79d3a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) - <0x00007efede1c0270> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff4dfb7898> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd559066d8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c86e598> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad1da880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 waiting for monitor entry [0x00007efbe5dd0000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4c70> (a org.apache.log4j.Logger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae739178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae889128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd58fbd6e8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae24f450> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff54272f80> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5bb2b5c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaee430d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f waiting on condition [0x00007efbff9f8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff0612eef0> (a java.lang.Object) - locked <0x00007eff0612eed0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 waiting on condition [0x00007efc054d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:501) - locked <0x00007eff8cd485e0> (a java.lang.Object) - locked <0x00007eff8cd485d0> (a java.lang.Object) at java.nio.channels.SocketChannel.write(SocketChannel.java:502) at org.apache.kafka.common.network.PlaintextTransportLayer.write(PlaintextTransportLayer.java:153) at org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:60) at org.apache.kafka.common.record.MultiRecordsSend.writeTo(MultiRecordsSend.java:93) at org.apache.kafka.common.network.KafkaChannel.send(KafkaChannel.java:339) at org.apache.kafka.common.network.KafkaChannel.write(KafkaChannel.java:310) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:512) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a waiting on condition [0x00007efc06d5a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 waiting on condition [0x00007efc0705d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c4a6128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: RUNNABLE at kafka.server.MetadataCache.kafka$server$MetadataCache$$getEndpoints(MetadataCache.scala:57) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1$$anonfun$apply$1.apply(MetadataCache.scala:78) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1$$anonfun$apply$1.apply(MetadataCache.scala:73) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1.apply(MetadataCache.scala:73) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1.apply(MetadataCache.scala:72) at scala.Option.map(Option.scala:146) at kafka.server.MetadataCache.kafka$server$MetadataCache$$getPartitionMetadata(MetadataCache.scala:72) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1$$anonfun$apply$7.apply(MetadataCache.scala:129) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1$$anonfun$apply$7.apply(MetadataCache.scala:128) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:241) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:104) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1.apply(MetadataCache.scala:128) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1.apply(MetadataCache.scala:128) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.MetadataCache.getTopicMetadata(MetadataCache.scala:127) at kafka.server.KafkaApis.getTopicMetadata(KafkaApis.scala:998) at kafka.server.KafkaApis.handleTopicMetadataRequest(KafkaApis.scala:1082) at kafka.server.KafkaApis.handle(KafkaApis.scala:109) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 waiting on condition [0x00007ef6aa1ba000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 waiting on condition [0x00007ef532e13000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 waiting on condition [0x00007ef533f12000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) - locked <0x00007efd832e97b0> (a java.lang.Object) at org.apache.kafka.common.network.Selector.wakeup(Selector.java:305) at kafka.network.Processor.wakeup(SocketServer.scala:870) at kafka.network.Processor.enqueueResponse(SocketServer.scala:829) at kafka.network.RequestChannel.sendResponse(RequestChannel.scala:336) at kafka.server.KafkaApis.kafka$server$KafkaApis$$sendResponse(KafkaApis.scala:2362) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2358) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: RUNNABLE at scala.collection.convert.Wrappers$JMapWrapperLike$$anon$2.next(Wrappers.scala:277) at scala.collection.convert.Wrappers$JMapWrapperLike$$anon$2.next(Wrappers.scala:274) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.KafkaApis.createResponse$1(KafkaApis.scala:615) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 runnable [0x00007efb0058e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efe7a3cf888> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 waiting on condition [0x00007efc63efd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.ConcurrentSkipListMap.cpr(ConcurrentSkipListMap.java:655) at java.util.concurrent.ConcurrentSkipListMap.findPredecessor(ConcurrentSkipListMap.java:682) at java.util.concurrent.ConcurrentSkipListMap.doPut(ConcurrentSkipListMap.java:823) at java.util.concurrent.ConcurrentSkipListMap.put(ConcurrentSkipListMap.java:1581) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricHolder.put(TimelineMetricsCache.java:154) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache.putTimelineMetric(TimelineMetricsCache.java:194) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheSanitizedTimelineMetric(KafkaTimelineMetricsReporter.java:480) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:418) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:15:21 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff9b1b3918> (a java.lang.Object) - locked <0x00007eff9b1b38f8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd59eedea0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a3d18> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a3d00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb049eff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at scala.collection.mutable.ResizableArray$class.$init$(ResizableArray.scala:32) at scala.collection.mutable.ArrayBuffer.(ArrayBuffer.scala:49) at scala.collection.mutable.ArrayBuffer.(ArrayBuffer.scala:63) at scala.collection.mutable.Buffer$.newBuilder(Buffer.scala:45) at scala.collection.generic.GenericTraversableTemplate$class.genericBuilder(GenericTraversableTemplate.scala:70) at scala.collection.AbstractTraversable.genericBuilder(Traversable.scala:104) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:57) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:52) at scala.collection.TraversableLike$class.builder$1(TraversableLike.scala:229) at scala.collection.TraversableLike$class.map(TraversableLike.scala:233) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$states(AbstractFetcherThread.scala:100) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914a8508> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c18e558> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efed0f614d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c2488d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) - <0x00007efed0f61a18> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "ReplicaFetcherThread-7-1064" #637 prio=5 os_prio=0 tid=0x00007ef65e78b000 nid=0x8b07 runnable [0x00007ef7d7748000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd88fd0048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a010f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a010e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01068> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.requests.FetchRequest$TopicAndPartitionData.batchByTopic(FetchRequest.java:248) at org.apache.kafka.common.requests.FetchRequest.toStruct(FetchRequest.java:467) at org.apache.kafka.common.requests.AbstractRequest.serialize(AbstractRequest.java:98) at org.apache.kafka.common.requests.AbstractRequest.toSend(AbstractRequest.java:91) at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:476) at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:450) at org.apache.kafka.clients.NetworkClient.send(NetworkClient.java:411) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:91) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at java.util.Arrays.copyOfRange(Arrays.java:3664) at java.lang.String.(String.java:207) at java.lang.StringBuilder.toString(StringBuilder.java:407) at java.util.Formatter.toString(Formatter.java:2355) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.partitionsToLogString(FetchSessionHandler.java:272) at org.apache.kafka.clients.FetchSessionHandler.access$200(FetchSessionHandler.java:55) at org.apache.kafka.clients.FetchSessionHandler$Builder.build(FetchSessionHandler.java:254) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:297) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae943fc8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at scala.collection.mutable.HashTable$class.$init$(HashTable.scala:48) at scala.collection.mutable.HashMap.(HashMap.scala:41) at scala.collection.mutable.HashMap.(HashMap.scala:56) at scala.collection.mutable.Map$.empty(Map.scala:61) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:955) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efed00255a8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b82a2d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839bff90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7aa2ecd0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913427c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ad1e70> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ad1e58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0acd150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e9ad30> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e9ad18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e96010> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04467e0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04467c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0441ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff01b2e228> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8b7089c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c22cd68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0201b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0201a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf01b498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff08e7c6a8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7804b150> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c229448> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe8075cdc8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b179278> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at java.util.regex.Matcher.(Matcher.java:225) at java.util.regex.Pattern.matcher(Pattern.java:1093) at java.util.Formatter.parse(Formatter.java:2547) at java.util.Formatter.format(Formatter.java:2501) at java.util.Formatter.format(Formatter.java:2455) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.responseDataToLogString(FetchSessionHandler.java:349) at org.apache.kafka.clients.FetchSessionHandler.handleResponse(FetchSessionHandler.java:424) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.ConcurrentHashMap.tabAt(ConcurrentHashMap.java:755) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:938) at kafka.utils.Pool.get(Pool.scala:66) at kafka.cluster.Partition.getReplica(Partition.scala:188) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:410) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:417) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:285) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:281) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:281) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c112628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff530c43e8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a056778> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d05b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0818e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae0818c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae07df48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3e7648> (a sun.nio.ch.Util$3) - locked <0x00007efdae3e7630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3e2928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effbc8ac920> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd509d24c0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efda9cdf438> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51) at sun.nio.ch.IOUtil.write(IOUtil.java:148) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:504) - locked <0x00007efdae59e1c8> (a java.lang.Object) at java.nio.channels.SocketChannel.write(SocketChannel.java:502) at org.apache.kafka.common.network.PlaintextTransportLayer.write(PlaintextTransportLayer.java:153) at org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:60) at org.apache.kafka.common.network.KafkaChannel.send(KafkaChannel.java:339) at org.apache.kafka.common.network.KafkaChannel.write(KafkaChannel.java:310) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:512) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d02510> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5b9d0fb8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1bc5d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe62a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01751b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb01751a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0170498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at scala.collection.mutable.ResizableArray$class.$init$(ResizableArray.scala:32) at scala.collection.mutable.ArrayBuffer.(ArrayBuffer.scala:49) at scala.collection.mutable.ArrayBuffer.(ArrayBuffer.scala:63) at scala.collection.mutable.Buffer$.newBuilder(Buffer.scala:45) at scala.collection.generic.GenericTraversableTemplate$class.genericBuilder(GenericTraversableTemplate.scala:70) at scala.collection.AbstractTraversable.genericBuilder(Traversable.scala:104) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:57) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:52) at scala.collection.TraversableLike$class.builder$1(TraversableLike.scala:229) at scala.collection.TraversableLike$class.map(TraversableLike.scala:233) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$states(AbstractFetcherThread.scala:100) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914e7a00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb37f4028> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d36ab30> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:253) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache.validateAndMaybeWarn(LeaderEpochFileCache.scala:200) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:66) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a34c840> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7a34cdd0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1e7d48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29540> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b294b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d724590> (a sun.nio.ch.Util$3) - locked <0x00007efe7d7245a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d724518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae065670> (a sun.nio.ch.Util$3) - locked <0x00007efdae065658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae060950> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb09decf8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa17a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a9a360> (a sun.nio.ch.Util$3) - locked <0x00007efda9a9a348> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a9a2d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac7f3120> (a sun.nio.ch.Util$3) - locked <0x00007efdac7f3108> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac7ee440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5a807408> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac644d50> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff1b8619e8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd83a82088> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: RUNNABLE at kafka.server.ReplicaManager.getLeaderReplicaIfLocal(ReplicaManager.scala:420) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$read$1(ReplicaManager.scala:896) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:989) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:988) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:988) at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:815) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:828) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007efdfc01f3d0> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.unpark(Native Method) at java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:141) at java.util.concurrent.locks.AbstractQueuedSynchronizer.unparkSuccessor(AbstractQueuedSynchronizer.java:662) at java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1264) at java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:457) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:422) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007eff35036178> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: RUNNABLE at kafka.server.ReplicaManager.getLeaderReplicaIfLocal(ReplicaManager.scala:420) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$read$1(ReplicaManager.scala:896) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:989) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:988) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:988) at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:815) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:828) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 waiting on condition [0x00007ef5383b3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007efdfc3e6ae0> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a runnable [0x00007efb00172000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efe7aa2ecd0> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 waiting on condition [0x00007efc63efd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.ConcurrentSkipListMap.findPredecessor(ConcurrentSkipListMap.java:691) at java.util.concurrent.ConcurrentSkipListMap.doGet(ConcurrentSkipListMap.java:781) at java.util.concurrent.ConcurrentSkipListMap.get(ConcurrentSkipListMap.java:1546) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricHolder.put(TimelineMetricsCache.java:152) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache.putTimelineMetric(TimelineMetricsCache.java:194) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheSanitizedTimelineMetric(KafkaTimelineMetricsReporter.java:480) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheKafkaMetered(KafkaTimelineMetricsReporter.java:427) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processMeter(KafkaTimelineMetricsReporter.java:365) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processMeter(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Meter.processWith(Meter.java:131) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:15:33 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #875 daemon prio=10 os_prio=0 tid=0x00007ef5c000a800 nid=0x1798 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c6674b0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007f04eb1a7330> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd59eedea0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe50a772c8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5bf3c780> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91358138> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6ed68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a3d18> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a3d00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb049eff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a75ef38> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9311a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed3c8fba8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78281640> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91470520> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd554118e0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdcc0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caafc08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdac18d6d0> (a java.lang.Object) - locked <0x00007efdac18d6b0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a3b9170> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e90308> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4e0908> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7acfced0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c2488d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf12f240> (a sun.nio.ch.Util$3) - locked <0x00007efdaf12f228> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf12a520> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7acd42c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91497860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7f6b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb498b3a8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe911f8c38> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839c8628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efef7924858> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe801334d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b36cf30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe800d64e0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe800d6a40> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe9144b4c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff4d371598> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a345f10> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839bff90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a2b1458> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7a2b22a8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe91458aa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c230740> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff08555ce0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efefe36b9d8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efed0bdeaa8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b927ba8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe786132b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff6cd65da0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effc84ca2b8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe805bccb0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7e5c98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf0160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c638680> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at scala.collection.mutable.Buffer$.newBuilder(Buffer.scala:45) at scala.collection.generic.GenericTraversableTemplate$class.genericBuilder(GenericTraversableTemplate.scala:70) at scala.collection.AbstractTraversable.genericBuilder(Traversable.scala:104) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:57) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:52) at scala.collection.TraversableLike$class.builder$1(TraversableLike.scala:229) at scala.collection.TraversableLike$class.map(TraversableLike.scala:233) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$states(AbstractFetcherThread.scala:100) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7e9ca0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff0859c4d8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.Selector.poll(Selector.java:429) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efda9cdf438> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b356ce0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:120) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb5ce9b40> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5bfa4850> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80da6288> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e134c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80dc89d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaf672230> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe915251f0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed6045d90> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a3b48f0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe62a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effbc3e9dc8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a3066c0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88fbc6e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c62c958> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7d387330> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb06e3918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff61a387c8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78c885e0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf0c2788> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd50d79b90> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0054380> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe78a03ae8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78a04078> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff20fc8e58> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd508425a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb00ab6a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effbc9be420> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5521baf0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910b4128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:43) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5dfa52c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6bf08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c19caf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe4d48f348> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff13b1b6e0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb5acfa20> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd51b28528> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80bfbc98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0945ec0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.internals.PartitionStates.partitionStates(PartitionStates.java:83) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$states(AbstractFetcherThread.scala:100) at kafka.server.AbstractFetcherThread$$anonfun$3.apply(AbstractFetcherThread.scala:127) at kafka.server.AbstractFetcherThread$$anonfun$3.apply(AbstractFetcherThread.scala:127) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.maybeTruncate(AbstractFetcherThread.scala:127) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9738a428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d724590> (a sun.nio.ch.Util$3) - locked <0x00007efe7d7245a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d724518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed4b62128> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efed0ea1b90> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae7e7f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7d336f78> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb08d5160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb9242370> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd506b1b58> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb09decf8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91696b48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd50eda2d8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8cb21e08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad22cb38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5e5b2490> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd5e5e4b00> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efdad50e370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad7cb4e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efec8a5b690> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd557b9870> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb075d8c0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at java.util.regex.Matcher.(Matcher.java:225) at java.util.regex.Pattern.matcher(Pattern.java:1093) at java.util.Formatter.parse(Formatter.java:2547) at java.util.Formatter.format(Formatter.java:2501) at java.util.Formatter.format(Formatter.java:2455) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.responseDataToLogString(FetchSessionHandler.java:349) at org.apache.kafka.clients.FetchSessionHandler.handleResponse(FetchSessionHandler.java:424) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff5f83ea00> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2e1020> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafd83f70> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e80910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe780a64e8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf5750b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) - <0x00007efe780a6ad0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac7f3120> (a sun.nio.ch.Util$3) - locked <0x00007efdac7f3108> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac7ee440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5a807408> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac644d50> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff4d2f0f10> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe783216f0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c20e18> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effaf61d5a0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d0913a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad1da880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:253) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe801992e0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaea48250> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) - <0x00007efe80199858> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91d8b7f8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1c1118> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.Selector.poll(Selector.java:389) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd82f6feb8> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.ConcurrentSkipListMap.cpr(ConcurrentSkipListMap.java:655) at java.util.concurrent.ConcurrentSkipListMap.doPut(ConcurrentSkipListMap.java:835) at java.util.concurrent.ConcurrentSkipListMap.putIfAbsent(ConcurrentSkipListMap.java:1979) at com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:104) at com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:81) at com.yammer.metrics.core.Histogram.update(Histogram.java:110) at kafka.network.RequestChannel$Request$$anonfun$updateRequestMetrics$1.apply(RequestChannel.scala:167) at kafka.network.RequestChannel$Request$$anonfun$updateRequestMetrics$1.apply(RequestChannel.scala:159) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.network.RequestChannel$Request.updateRequestMetrics(RequestChannel.scala:159) at kafka.network.Processor.kafka$network$Processor$$updateRequestMetrics(SocketServer.scala:740) at kafka.network.Processor$$anonfun$processCompletedSends$1.apply(SocketServer.scala:720) at kafka.network.Processor$$anonfun$processCompletedSends$1.apply(SocketServer.scala:715) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedSends(SocketServer.scala:715) at kafka.network.Processor.run(SocketServer.scala:585) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at java.util.LinkedHashMap.newNode(LinkedHashMap.java:256) at java.util.HashMap.putVal(HashMap.java:642) at java.util.HashMap.put(HashMap.java:612) at org.apache.kafka.common.requests.FetchRequest.(FetchRequest.java:399) at org.apache.kafka.common.requests.AbstractRequest.parseRequest(AbstractRequest.java:144) at org.apache.kafka.common.requests.RequestContext.parseRequest(RequestContext.java:64) at kafka.network.RequestChannel$Request.(RequestChannel.scala:89) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:696) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 waiting on condition [0x00007efc05a3c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b waiting on condition [0x00007efc06c59000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 waiting on condition [0x00007ef6aa1ba000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe915251f0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.partitionCount(AbstractFetcherThread.scala:392) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:159) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:158) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.server.AbstractFetcherManager.shutdownIdleFetcherThreads(AbstractFetcherManager.scala:158) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1130) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f waiting on condition [0x00007ef607476000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.SessionlessFetchContext$$anonfun$foreachPartition$1.apply(FetchSession.scala:331) at kafka.server.SessionlessFetchContext$$anonfun$foreachPartition$1.apply(FetchSession.scala:331) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.SessionlessFetchContext.foreachPartition(FetchSession.scala:331) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 waiting on condition [0x00007ef53851b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: RUNNABLE at kafka.server.ReplicaManager.getLeaderReplicaIfLocal(ReplicaManager.scala:420) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$read$1(ReplicaManager.scala:896) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:989) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:988) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:988) at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:815) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:828) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 runnable [0x00007ef5f02fb000] java.lang.Thread.State: RUNNABLE at java.net.Inet4AddressImpl.lookupAllHostAddr(Native Method) at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:929) at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1324) at java.net.InetAddress.getAllByName0(InetAddress.java:1277) at java.net.InetAddress.getAllByName(InetAddress.java:1193) at java.net.InetAddress.getAllByName(InetAddress.java:1127) at java.net.InetAddress.getByName(InetAddress.java:1077) at java.net.InetSocketAddress.(InetSocketAddress.java:220) at sun.net.NetworkClient.doConnect(NetworkClient.java:175) at sun.net.www.http.HttpClient.openServer(HttpClient.java:463) at sun.net.www.http.HttpClient.openServer(HttpClient.java:558) - locked <0x00007f04eb69e328> (a sun.net.www.http.HttpClient) at sun.net.www.http.HttpClient.(HttpClient.java:242) at sun.net.www.http.HttpClient.New(HttpClient.java:339) at sun.net.www.http.HttpClient.New(HttpClient.java:357) at sun.net.www.protocol.http.HttpURLConnection.getNewHttpClient(HttpURLConnection.java:1220) at sun.net.www.protocol.http.HttpURLConnection.plainConnect0(HttpURLConnection.java:1156) at sun.net.www.protocol.http.HttpURLConnection.plainConnect(HttpURLConnection.java:1050) at sun.net.www.protocol.http.HttpURLConnection.connect(HttpURLConnection.java:984) at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1564) - locked <0x00007f04eb69e488> (a sun.net.www.protocol.http.HttpURLConnection) at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1492) - locked <0x00007f04eb69e488> (a sun.net.www.protocol.http.HttpURLConnection) at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:480) at com.sun.jersey.client.urlconnection.URLConnectionClientHandler._invoke(URLConnectionClientHandler.java:253) at com.sun.jersey.client.urlconnection.URLConnectionClientHandler.handle(URLConnectionClientHandler.java:153) at com.sun.jersey.api.client.Client.handle(Client.java:652) at com.sun.jersey.api.client.WebResource.handle(WebResource.java:682) at com.sun.jersey.api.client.WebResource.access$200(WebResource.java:74) at com.sun.jersey.api.client.WebResource$Builder.get(WebResource.java:509) at org.apache.ranger.admin.client.RangerAdminRESTClient$3.run(RangerAdminRESTClient.java:120) at org.apache.ranger.admin.client.RangerAdminRESTClient$3.run(RangerAdminRESTClient.java:113) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:360) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1710) at org.apache.ranger.admin.client.RangerAdminRESTClient.getServicePoliciesIfUpdated(RangerAdminRESTClient.java:123) at org.apache.ranger.plugin.util.PolicyRefresher.loadPolicyfromPolicyAdmin(PolicyRefresher.java:264) at org.apache.ranger.plugin.util.PolicyRefresher.loadPolicy(PolicyRefresher.java:202) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:171) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efd5dfa52c8> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 runnable [0x00007efb1d7fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 waiting on condition [0x00007efc63efd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50116b30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:209) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:285) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.offer(ScheduledThreadPoolExecutor.java:1010) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.add(ScheduledThreadPoolExecutor.java:1037) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.add(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ScheduledThreadPoolExecutor.reExecutePeriodic(ScheduledThreadPoolExecutor.java:346) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:296) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50058e70> (a java.util.concurrent.ThreadPoolExecutor$Worker) "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.unpark(Native Method) at java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:141) at java.util.concurrent.locks.AbstractQueuedSynchronizer.unparkSuccessor(AbstractQueuedSynchronizer.java:662) at java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1264) at java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:457) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1104) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricHolder.put(TimelineMetricsCache.java:149) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache.putTimelineMetric(TimelineMetricsCache.java:194) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheSanitizedTimelineMetric(KafkaTimelineMetricsReporter.java:480) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheKafkaMetered(KafkaTimelineMetricsReporter.java:427) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processMeter(KafkaTimelineMetricsReporter.java:365) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processMeter(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Meter.processWith(Meter.java:131) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:15:45 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #945 daemon prio=10 os_prio=0 tid=0x00007ef67c014000 nid=0x21c0 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:519) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effa23b87c8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efed0f743c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c6674b0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01911d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb01911b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb018c4b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.Selector.poll(Selector.java:429) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6ed68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 waiting for monitor entry [0x00007ef6ac311000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - locked <0x00007efd500d4c70> (a org.apache.log4j.Logger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9145af08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a3d18> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a3d00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb049eff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80cfbaf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at java.util.Formatter$FormatSpecifier.print(Formatter.java:3088) at java.util.Formatter$FormatSpecifier.print(Formatter.java:3083) at java.util.Formatter$FormatSpecifier.printInteger(Formatter.java:2787) at java.util.Formatter$FormatSpecifier.print(Formatter.java:2747) at java.util.Formatter.format(Formatter.java:2520) at java.util.Formatter.format(Formatter.java:2455) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.partitionsToLogString(FetchSessionHandler.java:272) at org.apache.kafka.clients.FetchSessionHandler.access$200(FetchSessionHandler.java:55) at org.apache.kafka.clients.FetchSessionHandler$Builder.build(FetchSessionHandler.java:253) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:297) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910d0ac0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe78a9bc80> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91470520> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe1027d508> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe911a1d00> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b17ec68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91497860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839c8628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:79) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe800dd478> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe800dd9a8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe9144bd00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb3769f50> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efede4400a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c231a78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe7850e6c0> (a java.lang.Object) - locked <0x00007efe7850e6a0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5ddec278> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd5de14e20> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe8c230740> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd506fcbf8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913427c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e9ad30> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e9ad18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e96010> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04467e0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04467c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0441ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at java.util.ArrayDeque.iterator(ArrayDeque.java:611) at org.apache.kafka.clients.InFlightRequests.hasExpiredRequest(InFlightRequests.java:166) at org.apache.kafka.clients.InFlightRequests.nodesWithTimedOutRequests(InFlightRequests.java:185) at org.apache.kafka.clients.NetworkClient.handleTimedOutRequests(NetworkClient.java:694) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:523) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd51a32948> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c22cd68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0201b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0201a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf01b498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.Selector.poll(Selector.java:429) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6400> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at java.lang.AbstractStringBuilder.(AbstractStringBuilder.java:68) at java.lang.StringBuffer.(StringBuffer.java:116) at java.text.DateFormat.format(DateFormat.java:345) at org.apache.log4j.helpers.PatternParser$DatePatternConverter.convert(PatternParser.java:443) at org.apache.log4j.helpers.PatternConverter.format(PatternConverter.java:65) at org.apache.log4j.PatternLayout.format(PatternLayout.java:506) at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:310) at org.apache.log4j.WriterAppender.append(WriterAppender.java:162) at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251) - locked <0x00007efd50182058> (a org.apache.log4j.ConsoleAppender) at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66) at org.apache.log4j.Category.callAppenders(Category.java:206) - locked <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9191e500> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d8bad8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b179278> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c638680> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d05b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912a0388> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff8cc8af00> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd58c417c0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c635498> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3e7648> (a sun.nio.ch.Util$3) - locked <0x00007efdae3e7630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3e2928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c5135c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe6ebd0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efdb0b07200> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a3ea640> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7a3eabb8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe915251f0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efebd3e2478> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5d4428d8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c62c958> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at java.util.Formatter$FormatSpecifier.print(Formatter.java:3088) at java.util.Formatter$FormatSpecifier.print(Formatter.java:3083) at java.util.Formatter$FormatSpecifier.printInteger(Formatter.java:2787) at java.util.Formatter$FormatSpecifier.print(Formatter.java:2747) at java.util.Formatter.format(Formatter.java:2520) at java.util.Formatter.format(Formatter.java:2455) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.partitionsToLogString(FetchSessionHandler.java:272) at org.apache.kafka.clients.FetchSessionHandler.access$200(FetchSessionHandler.java:55) at org.apache.kafka.clients.FetchSessionHandler$Builder.build(FetchSessionHandler.java:254) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:297) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c659790> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf181c70> (a sun.nio.ch.Util$3) - locked <0x00007efdaf181c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf17cf50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb52c8000> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efedebe36e8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff48a60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf732808> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80bfbc98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.Selector.poll(Selector.java:429) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.internals.PartitionStates.partitionStates(PartitionStates.java:82) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$states(AbstractFetcherThread.scala:100) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4cb60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff84076100> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7dbfcba0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae51d880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaeb32d70> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at java.util.HashMap$HashIterator.(HashMap.java:1433) at java.util.HashMap$KeyIterator.(HashMap.java:1467) at java.util.HashMap$KeySet.iterator(HashMap.java:917) at java.util.HashSet.iterator(HashSet.java:173) at sun.nio.ch.Util$3.iterator(Util.java:324) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:457) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa17a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad7cb4e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c230c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:120) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac7f3120> (a sun.nio.ch.Util$3) - locked <0x00007efdac7f3108> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac7ee440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae24f450> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1c1118> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe910afed0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaee430d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.unpark(Native Method) at java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:141) at java.util.concurrent.locks.AbstractQueuedSynchronizer.unparkSuccessor(AbstractQueuedSynchronizer.java:662) at java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1264) at java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:457) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:356) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd865b50b8> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff8cd05988> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c waiting for monitor entry [0x00007efc06542000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b waiting for monitor entry [0x00007efc06c59000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: RUNNABLE at kafka.server.MetadataCache.kafka$server$MetadataCache$$getAliveEndpoint(MetadataCache.scala:121) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getEndpoints$1.apply$mcVI$sp(MetadataCache.scala:58) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getEndpoints$1.apply(MetadataCache.scala:57) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getEndpoints$1.apply(MetadataCache.scala:57) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.MetadataCache.kafka$server$MetadataCache$$getEndpoints(MetadataCache.scala:57) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1$$anonfun$apply$1.apply(MetadataCache.scala:95) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1$$anonfun$apply$1.apply(MetadataCache.scala:73) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1.apply(MetadataCache.scala:73) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1.apply(MetadataCache.scala:72) at scala.Option.map(Option.scala:146) at kafka.server.MetadataCache.kafka$server$MetadataCache$$getPartitionMetadata(MetadataCache.scala:72) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1$$anonfun$apply$7.apply(MetadataCache.scala:129) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1$$anonfun$apply$7.apply(MetadataCache.scala:128) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:241) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:104) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1.apply(MetadataCache.scala:128) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1.apply(MetadataCache.scala:128) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.MetadataCache.getTopicMetadata(MetadataCache.scala:127) at kafka.server.KafkaApis.getTopicMetadata(KafkaApis.scala:998) at kafka.server.KafkaApis.handleTopicMetadataRequest(KafkaApis.scala:1082) at kafka.server.KafkaApis.handle(KafkaApis.scala:109) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 waiting for monitor entry [0x00007efc06e5b000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 waiting for monitor entry [0x00007efc0705d000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 waiting for monitor entry [0x00007efc0889e000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007effc867c4b0> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 waiting for monitor entry [0x00007ef6aa1ba000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: RUNNABLE at kafka.server.ReplicaManager.getLeaderReplicaIfLocal(ReplicaManager.scala:420) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$read$1(ReplicaManager.scala:896) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:989) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:988) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:988) at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:815) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:828) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e waiting for monitor entry [0x00007ef5f8dfe000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d waiting on condition [0x00007ef603e76000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.partitionCount(AbstractFetcherThread.scala:392) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:159) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:158) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.server.AbstractFetcherManager.shutdownIdleFetcherThreads(AbstractFetcherManager.scala:158) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1130) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b waiting for monitor entry [0x00007ef6ac10f000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a waiting for monitor entry [0x00007ef607f77000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 waiting for monitor entry [0x00007ef532e13000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 waiting for monitor entry [0x00007ef533c35000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 waiting for monitor entry [0x00007ef530153000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 waiting for monitor entry [0x00007ef533f12000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 waiting for monitor entry [0x00007ef5383b3000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 waiting for monitor entry [0x00007ef7d5223000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 waiting for monitor entry [0x00007efaf114e000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 waiting for monitor entry [0x00007efbe5bd9000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 waiting for monitor entry [0x00007ef530355000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - waiting to lock <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f waiting for monitor entry [0x00007ef538248000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.server.FetchManager.newContext(FetchSession.scala:759) - locked <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: RUNNABLE at java.lang.Throwable.fillInStackTrace(Native Method) at java.lang.Throwable.fillInStackTrace(Throwable.java:783) - locked <0x00007f054f3a8058> (a java.io.IOException) at java.lang.Throwable.(Throwable.java:265) at java.lang.Exception.(Exception.java:66) at java.io.IOException.(IOException.java:58) at org.apache.hadoop.security.Groups.noGroupsForUser(Groups.java:200) at org.apache.hadoop.security.Groups.getGroups(Groups.java:223) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$28.apply(KafkaApis.scala:1043) at kafka.server.KafkaApis$$anonfun$28.apply(KafkaApis.scala:1043) at scala.collection.TraversableLike$$anonfun$partition$1.apply(TraversableLike.scala:314) at scala.collection.TraversableLike$$anonfun$partition$1.apply(TraversableLike.scala:314) at scala.collection.immutable.Set$Set1.foreach(Set.scala:94) at scala.collection.TraversableLike$class.partition(TraversableLike.scala:314) at scala.collection.AbstractTraversable.partition(Traversable.scala:104) at kafka.server.KafkaApis.handleTopicMetadataRequest(KafkaApis.scala:1043) at kafka.server.KafkaApis.handle(KafkaApis.scala:109) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efe911a1d00> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 runnable [0x00007efb246ae000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 waiting on condition [0x00007efc63efd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.sanitizeName(KafkaTimelineMetricsReporter.java:502) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:414) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:15:56 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #945 daemon prio=10 os_prio=0 tid=0x00007ef67c014000 nid=0x21c0 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 runnable [0x00007ef61990d000] java.lang.Thread.State: RUNNABLE at kafka.log.Log$$anonfun$read$2.apply(Log.scala:1116) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.read(Log.scala:1116) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$read$1(ReplicaManager.scala:926) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:989) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:988) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:988) at kafka.server.DelayedFetch.onComplete(DelayedFetch.scala:148) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:70) at kafka.server.DelayedOperation.run(DelayedOperation.scala:143) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007eff8817fb08> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efdf2cd1220> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a547f98> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c932810> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01911d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb01911b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb018c4b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c20a718> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7acf1060> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91358138> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d32a38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff6a35d178> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff5b4efbb0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efee3ced7a8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b80f918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe78a9bc80> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91470520> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff51555b68> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a55d110> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4c4268> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a74dc50> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b940d58> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e90308> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c18e558> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb26ef4b8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efede18b090> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdaa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at java.util.regex.Matcher.(Matcher.java:225) at java.util.regex.Pattern.matcher(Pattern.java:1093) at java.util.Formatter.parse(Formatter.java:2547) at java.util.Formatter.format(Formatter.java:2501) at java.util.Formatter.format(Formatter.java:2455) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.responseDataToLogString(FetchSessionHandler.java:349) at org.apache.kafka.clients.FetchSessionHandler.handleResponse(FetchSessionHandler.java:424) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:242) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf12f240> (a sun.nio.ch.Util$3) - locked <0x00007efdaf12f228> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf12a520> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff774a15c0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe801388f8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b3a0648> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91497860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7f6b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c930078> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe807a7880> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff4a715b30> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed4a23428> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efede18a448> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c231a78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at scala.collection.mutable.ResizableArray$class.$init$(ResizableArray.scala:32) at scala.collection.mutable.ArrayBuffer.(ArrayBuffer.scala:49) at scala.collection.mutable.ArrayBuffer.(ArrayBuffer.scala:63) at scala.collection.mutable.Buffer$.newBuilder(Buffer.scala:45) at scala.collection.generic.GenericTraversableTemplate$class.genericBuilder(GenericTraversableTemplate.scala:70) at scala.collection.AbstractTraversable.genericBuilder(Traversable.scala:104) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:57) at scala.collection.generic.GenTraversableFactory$GenericCanBuildFrom.apply(GenTraversableFactory.scala:52) at scala.collection.TraversableLike$class.builder$1(TraversableLike.scala:229) at scala.collection.TraversableLike$class.map(TraversableLike.scala:233) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.AbstractFetcherThread.kafka$server$AbstractFetcherThread$$states(AbstractFetcherThread.scala:100) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80cf9340> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c230740> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e9ad30> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e9ad18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e96010> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04467e0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04467c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0441ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at scala.collection.generic.Growable$class.$plus$plus$eq(Growable.scala:59) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:104) at scala.collection.mutable.ArrayBuffer.$plus$plus$eq(ArrayBuffer.scala:48) at scala.collection.TraversableOnce$class.copyToBuffer(TraversableOnce.scala:275) at scala.collection.AbstractTraversable.copyToBuffer(Traversable.scala:104) at scala.collection.MapLike$class.toBuffer(MapLike.scala:326) at scala.collection.AbstractMap.toBuffer(Map.scala:59) at scala.collection.MapLike$class.toSeq(MapLike.scala:323) at scala.collection.AbstractMap.toSeq(Map.scala:59) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:245) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe8b7089c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c22cd68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0201b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0201a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf01b498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe786132b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd509c6e50> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c229448> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff81141f18> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a386aa8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c116f28> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effc867e210> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd50a40010> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b179278> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b923c78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c635498> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3e7648> (a sun.nio.ch.Util$3) - locked <0x00007efdae3e7630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3e2928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff681ad270> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a3855a8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c5135c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80da6288> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb0afc3e0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe83047ac0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e134c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a537d68> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1bc5d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe80992a68> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7dc701b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at java.util.Arrays.copyOfRange(Arrays.java:3664) at java.lang.String.(String.java:207) at java.lang.StringBuilder.toString(StringBuilder.java:407) at java.util.Formatter$FormatSpecifier.print(Formatter.java:3139) at java.util.Formatter$FormatSpecifier.print(Formatter.java:3083) at java.util.Formatter$FormatSpecifier.printInteger(Formatter.java:2787) at java.util.Formatter$FormatSpecifier.print(Formatter.java:2747) at java.util.Formatter.format(Formatter.java:2520) at java.util.Formatter.format(Formatter.java:2455) at java.lang.String.format(String.java:2940) at org.apache.kafka.clients.FetchSessionHandler.partitionsToLogString(FetchSessionHandler.java:272) at org.apache.kafka.clients.FetchSessionHandler.access$200(FetchSessionHandler.java:55) at org.apache.kafka.clients.FetchSessionHandler$Builder.build(FetchSessionHandler.java:255) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:297) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c18a050> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5ebd3d70> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff9f928> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb06e3918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.protocol.types.Schema.read(Schema.java:71) at org.apache.kafka.common.protocol.ApiKeys.parseResponse(ApiKeys.java:294) at org.apache.kafka.clients.NetworkClient.parseStructMaybeUpdateThrottleTimeMetrics(NetworkClient.java:637) at org.apache.kafka.clients.NetworkClient.handleCompletedReceives(NetworkClient.java:757) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:519) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe78087868> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae9e0910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe7c76ecb8> (a java.lang.Object) - locked <0x00007efe7c76ec98> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff334d29f0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd554f0f60> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe834f6c58> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effc6f51738> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2b3520> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe912320a8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c17fa60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab8eeae0> (a sun.nio.ch.Util$3) - locked <0x00007efdab8eeac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab8e9d28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff6225fa70> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff682059e0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d9e4880> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff48a60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff6c1e1900> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe7d85bd28> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c19caf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff6d92a1f0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8c63f1b8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0945ec0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe80be4f08> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff331df680> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8c646f90> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9738a428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5542e730> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae7e7f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae51d880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaeb32d70> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae065670> (a sun.nio.ch.Util$3) - locked <0x00007efdae065658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae060950> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:79) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5b86e440> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd5b910a60> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe8caa17a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:79) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a33e988> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac79d3a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) - <0x00007efe7a33eeb8> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd50745db8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91d8b7f8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed73f96b8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd55900a18> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae24f450> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at kafka.network.Processor.kafka$network$Processor$$updateRequestMetrics(SocketServer.scala:740) at kafka.network.Processor$$anonfun$processCompletedSends$1.apply(SocketServer.scala:720) at kafka.network.Processor$$anonfun$processCompletedSends$1.apply(SocketServer.scala:715) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedSends(SocketServer.scala:715) at kafka.network.Processor.run(SocketServer.scala:585) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f waiting on condition [0x00007efbff9f8000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.IOUtil.drain(Native Method) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:104) - locked <0x00007efd865ebfa8> (a java.lang.Object) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at java.lang.String.substring(String.java:1969) at java.lang.String.subSequence(String.java:2003) at java.util.regex.Matcher.getSubSequence(Matcher.java:1294) at java.util.regex.Matcher.group(Matcher.java:541) at org.apache.kafka.common.security.kerberos.KerberosName.parse(KerberosName.java:59) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.applyKerberosShortNamer(DefaultKafkaPrincipalBuilder.java:129) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.build(DefaultKafkaPrincipalBuilder.java:120) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.principal(SaslServerAuthenticator.java:286) at org.apache.kafka.common.network.KafkaChannel.principal(KafkaChannel.java:114) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:695) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.ConcurrentSkipListMap.cpr(ConcurrentSkipListMap.java:655) at java.util.concurrent.ConcurrentSkipListMap.findPredecessor(ConcurrentSkipListMap.java:682) at java.util.concurrent.ConcurrentSkipListMap.doRemove(ConcurrentSkipListMap.java:960) at java.util.concurrent.ConcurrentSkipListMap.remove(ConcurrentSkipListMap.java:1595) at com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:106) at com.yammer.metrics.stats.ExponentiallyDecayingSample.update(ExponentiallyDecayingSample.java:81) at com.yammer.metrics.core.Histogram.update(Histogram.java:110) at kafka.network.RequestChannel$Request$$anonfun$updateRequestMetrics$1.apply(RequestChannel.scala:167) at kafka.network.RequestChannel$Request$$anonfun$updateRequestMetrics$1.apply(RequestChannel.scala:159) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.network.RequestChannel$Request.updateRequestMetrics(RequestChannel.scala:159) at kafka.network.Processor.kafka$network$Processor$$updateRequestMetrics(SocketServer.scala:740) at kafka.network.Processor$$anonfun$processCompletedSends$1.apply(SocketServer.scala:720) at kafka.network.Processor$$anonfun$processCompletedSends$1.apply(SocketServer.scala:715) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedSends(SocketServer.scala:715) at kafka.network.Processor.run(SocketServer.scala:585) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007eff35032158> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 waiting on condition [0x00007efc08bdd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f waiting on condition [0x00007ef607476000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c waiting on condition [0x00007ef5f03fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3e134c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.partitionCount(AbstractFetcherThread.scala:392) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:159) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:158) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.server.AbstractFetcherManager.shutdownIdleFetcherThreads(AbstractFetcherManager.scala:158) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1130) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.requests.FetchResponse.addTopicData(FetchResponse.java:441) at org.apache.kafka.common.requests.FetchResponse.addResponseData(FetchResponse.java:433) at org.apache.kafka.common.requests.FetchResponse.toSend(FetchResponse.java:375) at org.apache.kafka.common.requests.RequestContext.buildResponse(RequestContext.java:78) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2350) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 runnable [0x00007efb1c231000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 runnable [0x00007efb1d346000] java.lang.Thread.State: RUNNABLE at java.io.UnixFileSystem.getBooleanAttributes0(Native Method) at java.io.UnixFileSystem.getBooleanAttributes(UnixFileSystem.java:242) at java.io.File.isFile(File.java:882) at kafka.log.ProducerStateManager$$anonfun$listSnapshotFiles$1$$anonfun$apply$2.apply(ProducerStateManager.scala:447) at kafka.log.ProducerStateManager$$anonfun$listSnapshotFiles$1$$anonfun$apply$2.apply(ProducerStateManager.scala:447) at scala.collection.TraversableLike$$anonfun$filterImpl$1.apply(TraversableLike.scala:248) at scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33) at scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:186) at scala.collection.TraversableLike$class.filterImpl(TraversableLike.scala:247) at scala.collection.TraversableLike$class.filter(TraversableLike.scala:259) at scala.collection.mutable.ArrayOps$ofRef.filter(ArrayOps.scala:186) at kafka.log.ProducerStateManager$$anonfun$listSnapshotFiles$1.apply(ProducerStateManager.scala:447) at kafka.log.ProducerStateManager$$anonfun$listSnapshotFiles$1.apply(ProducerStateManager.scala:446) at scala.Option.map(Option.scala:146) at kafka.log.ProducerStateManager$.listSnapshotFiles(ProducerStateManager.scala:446) at kafka.log.ProducerStateManager$.kafka$log$ProducerStateManager$$deleteSnapshotFiles(ProducerStateManager.scala:456) at kafka.log.ProducerStateManager$.deleteSnapshotsBefore(ProducerStateManager.scala:453) at kafka.log.ProducerStateManager.deleteSnapshotsBefore(ProducerStateManager.scala:762) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1568) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 runnable [0x00007efb246ae000] java.lang.Thread.State: RUNNABLE at java.nio.MappedByteBuffer.force0(Native Method) at java.nio.MappedByteBuffer.force(MappedByteBuffer.java:203) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.AbstractIndex.flush(AbstractIndex.scala:155) at kafka.log.LogSegment$$anonfun$flush$1.apply$mcV$sp(LogSegment.scala:467) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) at kafka.log.LogSegment.flush(LogSegment.scala:464) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1537) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1536) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50843828> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00007efe6fd6dda8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f runnable [0x00007efb3c1e2000] java.lang.Thread.State: RUNNABLE at scala.collection.TraversableLike$$anonfun$groupBy$1.apply(TraversableLike.scala:321) at scala.collection.TraversableLike$$anonfun$groupBy$1.apply(TraversableLike.scala:320) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike$class.groupBy(TraversableLike.scala:320) at scala.collection.AbstractTraversable.groupBy(Traversable.scala:104) at kafka.server.ReplicaManager.checkpointHighWatermarks(ReplicaManager.scala:1398) at kafka.server.ReplicaManager$$anonfun$1.apply$mcV$sp(ReplicaManager.scala:244) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506d7b50> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b waiting on condition [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a waiting on condition [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 waiting on condition [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.unpark(Native Method) at java.util.concurrent.locks.LockSupport.unpark(LockSupport.java:141) at java.util.concurrent.locks.AbstractQueuedSynchronizer.unparkSuccessor(AbstractQueuedSynchronizer.java:662) at java.util.concurrent.locks.AbstractQueuedSynchronizer.release(AbstractQueuedSynchronizer.java:1264) at java.util.concurrent.locks.ReentrantLock.unlock(ReentrantLock.java:457) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1104) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 waiting on condition [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50116b30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:209) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:285) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.offer(ScheduledThreadPoolExecutor.java:1010) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.add(ScheduledThreadPoolExecutor.java:1037) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.add(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ScheduledThreadPoolExecutor.reExecutePeriodic(ScheduledThreadPoolExecutor.java:346) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:296) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd5004e058> (a java.util.concurrent.ThreadPoolExecutor$Worker) "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 waiting on condition [0x00007efcb8341000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50044a58> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:16:07 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "Keep-Alive-Timer" #949 daemon prio=8 os_prio=0 tid=0x00007ef5e000c000 nid=0x22ad waiting on condition [0x00007ef7ee7fc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at sun.net.www.http.KeepAliveCache.run(KeepAliveCache.java:172) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #945 daemon prio=10 os_prio=0 tid=0x00007ef67c014000 nid=0x21c0 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c6674b0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01911d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb01911b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb018c4b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe509e6a78> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5ee26388> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6ed68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd559fb8b8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d32a38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c50eef0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caafc08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0362560> (a sun.nio.ch.Util$3) - locked <0x00007efdb0362548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb035d840> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff4c93ac20> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe8374a0b8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe43b02e78> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe801179b0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b17ec68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7f6b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7aa33e80> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac6e90b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe78c87990> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9144bd00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9144b4c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe0f1a90f8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at java.io.FileOutputStream.writeBytes(Native Method) at java.io.FileOutputStream.write(FileOutputStream.java:326) at sun.nio.cs.StreamEncoder.writeBytes(StreamEncoder.java:221) at sun.nio.cs.StreamEncoder.implFlushBuffer(StreamEncoder.java:291) at sun.nio.cs.StreamEncoder.implFlush(StreamEncoder.java:295) at sun.nio.cs.StreamEncoder.flush(StreamEncoder.java:141) - locked <0x00007effaab370e0> (a java.io.OutputStreamWriter) at java.io.OutputStreamWriter.flush(OutputStreamWriter.java:229) at org.apache.log4j.helpers.QuietWriter.flush(QuietWriter.java:59) at org.apache.log4j.WriterAppender.subAppend(WriterAppender.java:324) at org.apache.log4j.DailyRollingFileAppender.subAppend(DailyRollingFileAppender.java:369) at org.apache.log4j.WriterAppender.append(WriterAppender.java:162) at org.apache.log4j.AppenderSkeleton.doAppend(AppenderSkeleton.java:251) - locked <0x00007efd50146c78> (a org.apache.log4j.DailyRollingFileAppender) at org.apache.log4j.helpers.AppenderAttachableImpl.appendLoopOnAppenders(AppenderAttachableImpl.java:66) at org.apache.log4j.Category.callAppenders(Category.java:206) - locked <0x00007efd500d4c70> (a org.apache.log4j.Logger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.log.ProducerStateManager.info(ProducerStateManager.scala:480) at kafka.log.ProducerStateManager.takeSnapshot(ProducerStateManager.scala:665) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1489) - locked <0x00007efe7a5413d8> (a java.lang.Object) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1467) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.roll(Log.scala:1467) at kafka.log.Log.kafka$log$Log$$maybeRoll(Log.scala:1452) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:859) - locked <0x00007efe7a5413d8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf2d98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effadb76558> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a3cf888> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d04a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe786132b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at java.util.HashMap$KeySet.iterator(HashMap.java:917) at java.util.HashSet.iterator(HashSet.java:173) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:457) at org.apache.kafka.common.network.Selector.poll(Selector.java:429) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efefd6207e0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a0867b8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6400> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c116f28> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24b6e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24b6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24b650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff0859c4d8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c632650> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: RUNNABLE at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:305) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80204dc8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb16eb3c0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2b48e0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe6ebd0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7ac6d7e8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7dc701b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efebd538ec8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5ed0f5c0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe565a6d50> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7aa3b208> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeafe750> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:79) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe809e0100> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe809e0690> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe8caa4ab8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf181c70> (a sun.nio.ch.Util$3) - locked <0x00007efdaf181c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf17cf50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd59b641c0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c17fa60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5aebafa0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at scala.runtime.BoxesRunTime.boxToInteger(BoxesRunTime.java:65) at kafka.cluster.Partition.getReplica(Partition.scala:188) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:410) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:417) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:285) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:281) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:281) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c8a71e8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff13b1b700> (a java.lang.Object) - locked <0x00007eff13b1b6e0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff146d5828> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7829c1f0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80bfbc98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe80be4f08> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5be6ea38> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad9988d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb08d5160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a310898> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb09decf8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efff1e43780> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8b6949c0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaec6c9d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe8cb17248> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa17a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7d323888> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad7cb4e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at scala.runtime.BoxesRunTime.boxToInteger(BoxesRunTime.java:65) at kafka.cluster.Partition.getReplica(Partition.scala:188) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:410) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:417) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:285) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:281) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:281) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c11f498> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac79d3a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb34ac358> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a30fb98> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac644d50> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe8ca87c58> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1c1118> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 waiting on condition [0x00007efc054d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 waiting on condition [0x00007efc0889e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: RUNNABLE at scala.collection.IndexedSeqLike$class.iterator(IndexedSeqLike.scala:90) at scala.collection.mutable.ArrayBuffer.iterator(ArrayBuffer.scala:48) at scala.collection.convert.Wrappers$IterableWrapperTrait$class.iterator(Wrappers.scala:25) at scala.collection.convert.Wrappers$SeqWrapper.iterator(Wrappers.scala:65) at scala.collection.convert.Wrappers$SeqWrapper.iterator(Wrappers.scala:65) at org.apache.kafka.common.requests.MetadataResponse.toStruct(MetadataResponse.java:554) at org.apache.kafka.common.requests.AbstractResponse.serialize(AbstractResponse.java:41) at org.apache.kafka.common.requests.AbstractResponse.toSend(AbstractResponse.java:34) at org.apache.kafka.common.requests.RequestContext.buildResponse(RequestContext.java:78) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2350) at kafka.server.KafkaApis.sendResponseMaybeThrottle(KafkaApis.scala:2300) at kafka.server.KafkaApis.handleTopicMetadataRequest(KafkaApis.scala:1092) at kafka.server.KafkaApis.handle(KafkaApis.scala:109) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 waiting on condition [0x00007ef532e13000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c1c1118> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007eff35036498> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007eff4dd85ae0> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 runnable [0x00007efb1d346000] java.lang.Thread.State: RUNNABLE at java.io.FileDescriptor.sync(Native Method) at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:67) at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50) - locked <0x00007efd5089f708> (a java.lang.Object) at kafka.server.checkpoints.OffsetCheckpointFile.write(OffsetCheckpointFile.scala:59) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:581) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 runnable [0x00007efb244b4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 waiting on condition [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.toString(Integer.java:935) at java.lang.String.valueOf(String.java:2994) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:418) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:16:19 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #945 daemon prio=10 os_prio=0 tid=0x00007ef67c014000 nid=0x21c0 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe780311c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c6674b0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efed0dc33e0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a3d18> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a3d00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb049eff8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effae5c9920> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe800455a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9311a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b80f918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91470520> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4c4268> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.ConcurrentHashMap.tabAt(ConcurrentHashMap.java:755) at java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:938) at kafka.utils.Pool.get(Pool.scala:66) at kafka.cluster.Partition.getReplica(Partition.scala:188) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:410) at kafka.server.ReplicaManager.getReplicaOrException(ReplicaManager.scala:417) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:285) at kafka.server.ReplicaFetcherThread$$anonfun$buildFetchRequest$1.apply(ReplicaFetcherThread.scala:281) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaFetcherThread.buildFetchRequest(ReplicaFetcherThread.scala:281) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:105) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caafc08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd509ce6e0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e90308> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf12f240> (a sun.nio.ch.Util$3) - locked <0x00007efdaf12f228> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf12a520> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe780a58b0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7f6b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c930078> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04a4b20> (a sun.nio.ch.Util$3) - locked <0x00007efdb04a4b08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe800d64e0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9144b4c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5bcca810> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0c1f878> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04605e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04605d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb045d880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7a081348> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff08555ce0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0ba0cf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d04a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efef0757750> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a776170> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe786132b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88cac830> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7869daf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b923c78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff28f22db8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff530c43e8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a056778> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d05b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5a114450> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c635498> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c632650> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed5798938> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8034daa0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b356ce0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d02510> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5abfecd0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe915251f0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effbc9171f8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7dc32c78> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88fbc6e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7dc701b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe78c53220> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff4dfeaae0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8be727d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff9f928> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb05321f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb05321e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb052d4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0515188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0515170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0510468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007f05181b40e8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd55f70b60> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efed0bda8b0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf181c70> (a sun.nio.ch.Util$3) - locked <0x00007efdaf181c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf17cf50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a33f640> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c17fa60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab8eeae0> (a sun.nio.ch.Util$3) - locked <0x00007efdab8eeac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab8e9d28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb00bac88> (a sun.nio.ch.Util$3) - locked <0x00007efdb00bac70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb00b5fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:79) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5dfa52c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd5dfa8480> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe80c6bf08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c19caf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff650e6840> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe1028fae0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a3e8138> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0f4d810> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb71490> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb71478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb6c7b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d724590> (a sun.nio.ch.Util$3) - locked <0x00007efe7d7245a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d724518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb08d5160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff5f385998> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8c72de50> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaec6c9d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5abd1af8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91696b48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaec525d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaec525b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaec4d858> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff4a138158> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac7f3120> (a sun.nio.ch.Util$3) - locked <0x00007efdac7f3108> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac7ee440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdabf54b28> (a sun.nio.ch.Util$3) - locked <0x00007efdabf54b10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf4fe08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 waiting on condition [0x00007efc0471a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 runnable [0x00007efc05d3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685eec0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685eed8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f waiting on condition [0x00007efc05f41000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a waiting for monitor entry [0x00007efc06d5a000] java.lang.Thread.State: BLOCKED (on object monitor) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:205) - locked <0x00007efd83a82058> (a java.lang.Object) at org.apache.kafka.common.network.Selector.wakeup(Selector.java:305) at kafka.network.Processor.wakeup(SocketServer.scala:870) at kafka.network.Processor.enqueueResponse(SocketServer.scala:829) at kafka.network.RequestChannel.sendResponse(RequestChannel.scala:336) at kafka.server.KafkaApis.kafka$server$KafkaApis$$sendResponse(KafkaApis.scala:2362) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2358) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.SessionlessFetchContext$$anonfun$foreachPartition$1.apply(FetchSession.scala:331) at kafka.server.SessionlessFetchContext$$anonfun$foreachPartition$1.apply(FetchSession.scala:331) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.SessionlessFetchContext.foreachPartition(FetchSession.scala:331) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b waiting on condition [0x00007ef6ac10f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b356ce0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 waiting on condition [0x00007ef530153000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting for monitor entry [0x00007efb0058e000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.logSegments(Log.scala:1711) - waiting to lock <0x00007efd55f70b60> (a java.lang.Object) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd507462b0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 runnable [0x00007efb141d5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efe8be727d0> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 runnable [0x00007efb244b4000] java.lang.Thread.State: RUNNABLE at java.nio.MappedByteBuffer.force0(Native Method) at java.nio.MappedByteBuffer.force(MappedByteBuffer.java:203) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.AbstractIndex.flush(AbstractIndex.scala:155) at kafka.log.LogSegment$$anonfun$flush$1.apply$mcV$sp(LogSegment.scala:467) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) at kafka.log.LogSegment.flush(LogSegment.scala:464) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1537) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1536) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd508677c8> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00007effd3995a20> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f runnable [0x00007efb3c1e2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 waiting on condition [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at java.util.TreeMap.put(TreeMap.java:568) at java.util.AbstractMap.putAll(AbstractMap.java:281) at java.util.TreeMap.putAll(TreeMap.java:327) at org.apache.hadoop.metrics2.sink.timeline.TimelineMetric.addMetricValues(TimelineMetric.java:158) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricWrapper.putMetric(TimelineMetricsCache.java:97) - locked <0x00007effe823ff90> (a org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricWrapper) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricHolder.put(TimelineMetricsCache.java:156) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache.putTimelineMetric(TimelineMetricsCache.java:194) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheSanitizedTimelineMetric(KafkaTimelineMetricsReporter.java:480) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:418) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:16:31 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #945 daemon prio=10 os_prio=0 tid=0x00007ef67c014000 nid=0x21c0 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01911d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb01911b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb018c4b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff0240af48> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff31117190> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5e6add60> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d32a38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b runnable [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac1a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a54b3a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91470520> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caafc08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0411ad0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0411ab8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf83e3a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb042c4a8> (a sun.nio.ch.Util$3) - locked <0x00007efdb042c490> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0427788> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efeab78d7d8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04f95f8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04f95e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04f48d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 runnable [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf runnable [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b82a2d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed40c3ef8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe80130240> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9144b4c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effa2043608> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd506fcbf8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913427c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e9ad30> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e9ad18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e96010> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd runnable [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04467e0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04467c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0441ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed41b7470> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7aa37fb8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d04a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d33b88> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d33b70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d2ee68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3935c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3935b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad8a4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ff25a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab90d268> (a sun.nio.ch.Util$3) - locked <0x00007efdab90d250> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab908548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04dbae0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04dbac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04d6dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0479ec8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0479eb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04751a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0805b60> (a sun.nio.ch.Util$3) - locked <0x00007efdb0805b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0800e40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effc867e210> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd50a40010> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b179278> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c27cc50> (a sun.nio.ch.Util$3) - locked <0x00007efe8c27cc38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c27c940> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c638680> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff31b88808> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a5562a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d05b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04906e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0818e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae0818c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae07df48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3e7648> (a sun.nio.ch.Util$3) - locked <0x00007efdae3e7630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3e2928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5bded0b0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe6ebd0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2df6b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2df6a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed57408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0b07160> (a sun.nio.ch.Util$3) - locked <0x00007efdb0b07148> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0b02440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0cfbdb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0cfbda0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0cf7098> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be22b0> (a sun.nio.ch.Util$3) - locked <0x00007efe78be22c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be2238> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb06e3918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0ab6440> (a sun.nio.ch.Util$3) - locked <0x00007efdb0ab6428> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ab1720> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effbc9c3a08> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd506fbea0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe916a53c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0389988> (a sun.nio.ch.Util$3) - locked <0x00007efdb0389970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0384c68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafad2c98> (a sun.nio.ch.Util$3) - locked <0x00007efdafad2c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafacdfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf319970> (a sun.nio.ch.Util$3) - locked <0x00007efdaf319958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf314c50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab8eeae0> (a sun.nio.ch.Util$3) - locked <0x00007efdab8eeac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab8e9d28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb089bf80> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff6225fa90> (a java.lang.Object) - locked <0x00007eff6225fa70> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb037bcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdb037bca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0376fa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1e7d48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 runnable [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb34ecf58> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe910e8808> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf732808> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaeb6ccf0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdaba058c0> (a java.lang.Object) - locked <0x00007efdaba058a0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d724590> (a sun.nio.ch.Util$3) - locked <0x00007efe7d7245a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d724518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae7e7f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad9988d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae065670> (a sun.nio.ch.Util$3) - locked <0x00007efdae065658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae060950> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe804c3ee0> (a sun.nio.ch.Util$3) - locked <0x00007efe804c3ef8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe804c3e68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:79) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a341cc0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7a342328> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efe91696b48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:120) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04c02b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04c0298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04bb590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb51780c0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2bbfd0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c230c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0faebd0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0faebb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fa9eb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0959860> (a sun.nio.ch.Util$3) - locked <0x00007efdb0959848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff7af0ccc0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff4d2f0f10> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe783216f0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c20e18> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae889128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd832eba30> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at java.util.regex.Matcher.(Matcher.java:225) at java.util.regex.Pattern.matcher(Pattern.java:1093) at org.apache.kafka.common.security.kerberos.KerberosRule.replaceParameters(KerberosRule.java:120) at org.apache.kafka.common.security.kerberos.KerberosRule.apply(KerberosRule.java:180) at org.apache.kafka.common.security.kerberos.KerberosShortNamer.shortName(KerberosShortNamer.java:93) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.applyKerberosShortNamer(DefaultKafkaPrincipalBuilder.java:131) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.build(DefaultKafkaPrincipalBuilder.java:120) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.principal(SaslServerAuthenticator.java:286) at org.apache.kafka.common.network.KafkaChannel.principal(KafkaChannel.java:114) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:695) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at java.nio.charset.Charset.forName(Charset.java:528) at java.net.URLEncoder.encode(URLEncoder.java:212) at org.apache.kafka.common.utils.Sanitizer.sanitize(Sanitizer.java:55) at kafka.network.RequestChannel$Session.(RequestChannel.scala:52) at kafka.network.RequestChannel$Request.(RequestChannel.scala:88) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:696) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd832ea470> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc980> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc968> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc998> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 waiting on condition [0x00007efc05d3f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a waiting on condition [0x00007efc06d5a000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007effc867ccc0> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 waiting on condition [0x00007efc08bdd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe916a53c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007eff35036178> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b waiting on condition [0x00007ef6ac10f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) - locked <0x00007efd86672480> (a java.lang.Object) at org.apache.kafka.common.network.Selector.wakeup(Selector.java:305) at kafka.network.Processor.wakeup(SocketServer.scala:870) at kafka.network.Processor.enqueueResponse(SocketServer.scala:829) at kafka.network.RequestChannel.sendResponse(RequestChannel.scala:336) at kafka.server.KafkaApis.kafka$server$KafkaApis$$sendResponse(KafkaApis.scala:2362) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2358) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 waiting on condition [0x00007ef533f12000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:273) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 runnable [0x00007efb141d5000] java.lang.Thread.State: RUNNABLE at java.nio.MappedByteBuffer.force0(Native Method) at java.nio.MappedByteBuffer.force(MappedByteBuffer.java:203) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.log.AbstractIndex$$anonfun$flush$1.apply(AbstractIndex.scala:156) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.AbstractIndex.flush(AbstractIndex.scala:155) at kafka.log.LogSegment$$anonfun$flush$1.apply$mcV$sp(LogSegment.scala:466) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.log.LogSegment$$anonfun$flush$1.apply(LogSegment.scala:464) at kafka.metrics.KafkaTimer.time(KafkaTimer.scala:31) at kafka.log.LogSegment.flush(LogSegment.scala:464) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1537) at kafka.log.Log$$anonfun$flush$1$$anonfun$apply$mcV$sp$6.apply(Log.scala:1536) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506fda08> (a java.util.concurrent.ThreadPoolExecutor$Worker) - <0x00007efeed2550a0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 runnable [0x00007efb1c231000] java.lang.Thread.State: RUNNABLE at sun.nio.fs.UnixNativeDispatcher.rename0(Native Method) at sun.nio.fs.UnixNativeDispatcher.rename(UnixNativeDispatcher.java:183) at sun.nio.fs.UnixCopyFile.move(UnixCopyFile.java:388) at sun.nio.fs.UnixFileSystemProvider.move(UnixFileSystemProvider.java:262) at java.nio.file.Files.move(Files.java:1395) at org.apache.kafka.common.utils.Utils.atomicMoveWithFallback(Utils.java:783) at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:72) at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50) - locked <0x00007efd83065540> (a java.lang.Object) at kafka.server.checkpoints.OffsetCheckpointFile.write(OffsetCheckpointFile.scala:59) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$50.apply(ReplicaManager.scala:1402) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2$$anonfun$apply$50.apply(ReplicaManager.scala:1402) at scala.Option.foreach(Option.scala:257) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1402) at kafka.server.ReplicaManager$$anonfun$checkpointHighWatermarks$2.apply(ReplicaManager.scala:1399) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.immutable.HashMap$HashMap1.foreach(HashMap.scala:221) at scala.collection.immutable.HashMap$HashTrieMap.foreach(HashMap.scala:428) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.server.ReplicaManager.checkpointHighWatermarks(ReplicaManager.scala:1399) at kafka.server.ReplicaManager$$anonfun$1.apply$mcV$sp(ReplicaManager.scala:244) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506b46e0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 runnable [0x00007efb1d346000] java.lang.Thread.State: RUNNABLE at java.io.FileDescriptor.sync(Native Method) at kafka.server.checkpoints.CheckpointFile.liftedTree1$1(CheckpointFile.scala:67) at kafka.server.checkpoints.CheckpointFile.write(CheckpointFile.scala:50) - locked <0x00007efd5089cd50> (a java.lang.Object) at kafka.server.checkpoints.OffsetCheckpointFile.write(OffsetCheckpointFile.scala:59) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:581) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 runnable [0x00007efb1d7fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 waiting on condition [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.sanitizeName(KafkaTimelineMetricsReporter.java:500) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:414) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:16:43 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #945 daemon prio=10 os_prio=0 tid=0x00007ef67c014000 nid=0x21c0 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "process reaper" #929 daemon prio=10 os_prio=0 tid=0x00007ef6a000d800 nid=0x1f4d waiting on condition [0x00007efc61eb4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 runnable [0x00007ef61990d000] java.lang.Thread.State: RUNNABLE at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:587) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.DelayedFetch.onComplete(DelayedFetch.scala:163) at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:70) at kafka.server.DelayedOperation.run(DelayedOperation.scala:143) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007eff8817fb08> (a java.util.concurrent.ThreadPoolExecutor$Worker) "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c932810> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 waiting for monitor entry [0x00007ef651d14000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b532fb8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0338040> (a sun.nio.ch.Util$3) - locked <0x00007efdb0338028> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0333360> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c runnable [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b waiting for monitor entry [0x00007ef6621fe000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb01adb00> (a sun.nio.ch.Util$3) - locked <0x00007efdb01adae8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01a8de0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efed0becd18> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 runnable [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd509d2b50> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91358138> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 runnable [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe976a27e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5e5230> (a sun.nio.ch.Util$3) - locked <0x00007efdae5e5218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5e0510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d32a38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d waiting for monitor entry [0x00007ef6e0ae8000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c89fd20> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 runnable [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dabf90> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 runnable [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 waiting for monitor entry [0x00007ef71f9fe000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80cfbaf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 waiting for monitor entry [0x00007ef7266fd000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b162ad0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f waiting for monitor entry [0x00007ef72706b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1e8ba0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae41f750> (a sun.nio.ch.Util$3) - locked <0x00007efdae41f738> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae41aa30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c runnable [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac160> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b waiting for monitor entry [0x00007ef743fc4000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91355a78> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91470520> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d53d50> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d53d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d4f030> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 waiting for monitor entry [0x00007ef7b85ba000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b80d150> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effc64d02d8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe91167a48> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b940d58> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260ef0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d938198> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d937e70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07aa4b0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07aa498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07a5790> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07ea8a0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07ea888> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07e5b80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd59ed3c18> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c2488d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 runnable [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 waiting for monitor entry [0x00007ef7da201000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91aa5030> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0f7a000> (a sun.nio.ch.Util$3) - locked <0x00007efdb0f79fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0f752e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 runnable [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac420> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe782eb438> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdaa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd runnable [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a7f9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a7f9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb040f798> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb runnable [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1035" #624 prio=5 os_prio=0 tid=0x00007ef65e62c800 nid=0x8afa runnable [0x00007ef7ee4f9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b5288b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at kafka.log.LogSegment.append(LogSegment.scala:139) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2f6500> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b17ec68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 waiting for monitor entry [0x00007ef7ee8fd000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b3a0648> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 waiting for monitor entry [0x00007ef7eeaff000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91497860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef waiting for monitor entry [0x00007ef7f2480000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7f6b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed waiting for monitor entry [0x00007ef7f2682000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9144d390> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 waiting for monitor entry [0x00007ef80670e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7d902328> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 runnable [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efee3dac528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac6e90b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 runnable [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0d0a78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0d0a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf0cbd58> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf waiting for monitor entry [0x00007ef818e42000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80e06e10> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add waiting for monitor entry [0x00007ef819044000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91012370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 waiting for monitor entry [0x00007ef81b5ee000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b82a2d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed4c76a70> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2c1a40> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b36cf30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c022990> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 runnable [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 waiting for monitor entry [0x00007ef820357000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b282288> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e62188> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e62170> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e5d468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf waiting for monitor entry [0x00007ef823536000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0c3bca8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd waiting for monitor entry [0x00007ef823983000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9144b4c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf36f728> (a sun.nio.ch.Util$3) - locked <0x00007efdaf36f710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae709838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 waiting for monitor entry [0x00007ef8883ef000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c230740> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913427c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e9ad30> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e9ad18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e96010> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd waiting for monitor entry [0x00007ef890689000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c76d388> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04467e0> (a sun.nio.ch.Util$3) - locked <0x00007efdb04467c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0441ac0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 waiting for monitor entry [0x00007ef89c2e3000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.log.ProducerStateManager.info(ProducerStateManager.scala:480) at kafka.log.ProducerStateManager.takeSnapshot(ProducerStateManager.scala:665) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1489) - locked <0x00007efd5e72fff0> (a java.lang.Object) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1467) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.roll(Log.scala:1467) at kafka.log.Log.kafka$log$Log$$maybeRoll(Log.scala:1452) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:859) - locked <0x00007efd5e72fff0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0ba0cf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efefde6e828> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78088ae8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c22cd68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 waiting for monitor entry [0x00007ef89c6f3000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d04a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe80a0aa30> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b927ba8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 runnable [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 waiting for monitor entry [0x00007ef8bd84f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1f1ef0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff1bd0b258> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 waiting for monitor entry [0x00007ef8c2b30000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9197eb20> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf runnable [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac waiting for monitor entry [0x00007ef8c6112000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912a2820> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03c0100> (a sun.nio.ch.Util$3) - locked <0x00007efdb03c00e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03bb3e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa waiting for monitor entry [0x00007ef8cdd08000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c63b020> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 waiting for monitor entry [0x00007ef8ced42000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6400> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007f04e90e8018> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7d3053c0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c229448> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 waiting for monitor entry [0x00007ef8dbefd000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78664898> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 waiting for monitor entry [0x00007ef8dbffe000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe916b35b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe805b6808> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c116f28> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff47b58f58> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 runnable [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f waiting for monitor entry [0x00007ef8dc402000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9191e500> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e runnable [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe789a7a08> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d8bad8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a waiting for monitor entry [0x00007ef8e45dc000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910b4398> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 waiting for monitor entry [0x00007ef8e49e0000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe919959a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 waiting for monitor entry [0x00007ef8e4ce3000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b179278> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 waiting for monitor entry [0x00007ef8fad2b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7e5c98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 runnable [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e7dcb8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e7dca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e78f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e waiting for monitor entry [0x00007ef910ac6000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d28408> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c waiting for monitor entry [0x00007ef910cc8000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913f0500> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5ed4dc00> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf0160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 runnable [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf98c380> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcd8608> (a sun.nio.ch.Util$3) - locked <0x00007efdafcd85f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf36bd48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb012b210> (a sun.nio.ch.Util$3) - locked <0x00007efdb012b1f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01264f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e21f68> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e21f50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e1d248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f runnable [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d waiting for monitor entry [0x00007ef919a0d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8ba534e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03a4df8> (a sun.nio.ch.Util$3) - locked <0x00007efdb03a4de0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03a00d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d16420> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d16408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d11700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 waiting for monitor entry [0x00007ef952be4000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7e9ca0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 waiting for monitor entry [0x00007ef952de6000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912a0388> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 runnable [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff0859c4d8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe8034e4d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c632650> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 runnable [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c1ebf0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efea1ba2668> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd55afa820> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c5135c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e runnable [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at kafka.log.LogSegment.append(LogSegment.scala:139) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe789edfa8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe6ebd0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff62130e08> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b356ce0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 waiting for monitor entry [0x00007ef990753000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - locked <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafcc9c68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 runnable [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0c02790> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 waiting for monitor entry [0x00007ef993e54000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb00d22b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 waiting for monitor entry [0x00007ef993f55000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf95cdc0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe8b6f4da8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80da6288> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0be5d48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaf198d38> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e134c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 waiting for monitor entry [0x00007ef9c1912000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c183008> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 runnable [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efef0a4da08> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a34d5a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe62a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fca620> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fca608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fc5900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a runnable [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff1ba53240> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0fe3d10> (a sun.nio.ch.Util$3) - locked <0x00007efdb0fe3cf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0fdeff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe78c53220> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 waiting for monitor entry [0x00007efa0ac80000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafcc6678> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at java.util.ArrayDeque.iterator(ArrayDeque.java:611) at org.apache.kafka.clients.InFlightRequests.hasExpiredRequest(InFlightRequests.java:166) at org.apache.kafka.clients.InFlightRequests.nodesWithTimedOutRequests(InFlightRequests.java:185) at org.apache.kafka.clients.NetworkClient.handleTimedOutRequests(NetworkClient.java:694) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:523) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 waiting for monitor entry [0x00007efa1f9e8000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf0c2788> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at kafka.log.LogSegment.append(LogSegment.scala:139) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5a120690> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae9e0910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 waiting for monitor entry [0x00007efa2211b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80dc6300> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 waiting for monitor entry [0x00007efa22326000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c659790> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe916a53c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 waiting for monitor entry [0x00007efa28c5f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf8f1a60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 waiting for monitor entry [0x00007efa28d60000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 runnable [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafad2c98> (a sun.nio.ch.Util$3) - locked <0x00007efdafad2c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafacdfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d runnable [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b waiting for monitor entry [0x00007efa4029d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa4ab8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:79) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efee3ca6780> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe4d550> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) - <0x00007efee3ca6e58> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 waiting for monitor entry [0x00007efa40cca000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9148b418> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7d08ca88> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c17fa60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab8eeae0> (a sun.nio.ch.Util$3) - locked <0x00007efdab8eeac8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab8e9d28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0aec840> (a sun.nio.ch.Util$3) - locked <0x00007efdb0aec828> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0ae7b20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e waiting for monitor entry [0x00007efa643f0000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb089bf50> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d waiting for monitor entry [0x00007efa644f1000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe976dddc8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe800cbc08> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb00ab6a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b waiting for monitor entry [0x00007efa7083b000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910b4128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb04984e8> (a sun.nio.ch.Util$3) - locked <0x00007efdb04984d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb04937c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80c6bf08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c19caf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 waiting for monitor entry [0x00007efa7d57c000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cc7b0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 runnable [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 waiting for monitor entry [0x00007efa7df39000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c8a71e8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf732808> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 waiting for monitor entry [0x00007efab46d4000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0945ec0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efe80be4f08> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9738a428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd5de95c98> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeb51428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 waiting for monitor entry [0x00007efabc950000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcead08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:111) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7d4d2558> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03fa110> (a sun.nio.ch.Util$3) - locked <0x00007efdb03fa0f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03f53f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaba0a0d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaba0a0c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaba05320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd59e99790> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 waiting for monitor entry [0x00007efad7c96000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafe303c0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae47ed98> (a sun.nio.ch.Util$3) - locked <0x00007efdae47ed80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae47a078> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaeb32d70> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df waiting for monitor entry [0x00007efae544a000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb08d5160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de waiting for monitor entry [0x00007efaf174e000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaea8ee30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc waiting for monitor entry [0x00007efaf2bfe000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb09decf8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da runnable [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 waiting for monitor entry [0x00007efafb7f6000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae990c18> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91696b48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0f4afa0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce runnable [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd waiting for monitor entry [0x00007efb25f7d000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdb0deaaf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc waiting for monitor entry [0x00007efb7c5fc000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c187918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a1efa0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a1ef88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a1a2c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 waiting for monitor entry [0x00007efb882fd000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf998560> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 runnable [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdb0eeb3c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 waiting for monitor entry [0x00007efbb829f000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac811e90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 runnable [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0d6f130> (a sun.nio.ch.Util$3) - locked <0x00007efdb0d6f118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0d6a410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdb04bbb10> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae49a048> (a sun.nio.ch.Util$3) - locked <0x00007efdae49a030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af waiting for monitor entry [0x00007efbbca17000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.log.Log.info(Log.scala:158) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1508) - locked <0x00007efe78024920> (a java.lang.Object) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1467) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.roll(Log.scala:1467) at kafka.log.Log.kafka$log$Log$$maybeRoll(Log.scala:1452) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:859) - locked <0x00007efe78024920> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe807d0f60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad waiting for monitor entry [0x00007efbbce00000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c230c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.Selector.poll(Selector.java:429) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb07c9ab0> (a sun.nio.ch.Util$3) - locked <0x00007efdb07c9a98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb07c4d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa runnable [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 waiting for monitor entry [0x00007efbc83fe000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae773f88> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdafd83f70> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 runnable [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 waiting for monitor entry [0x00007efbd65fa000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efee3e80910> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 waiting for monitor entry [0x00007efbd68fb000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf5750b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f runnable [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac79d3a8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a waiting for monitor entry [0x00007efbd7295000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae773a00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 waiting for monitor entry [0x00007efbe41f3000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c86e598> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac644d50> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe78085050> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c20e18> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 runnable [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 waiting for monitor entry [0x00007efbe5a11000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7863d958> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0a9b4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdb0a9b4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0a967b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0e45a58> (a sun.nio.ch.Util$3) - locked <0x00007efdb0e45a40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb0e40d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb03db368> (a sun.nio.ch.Util$3) - locked <0x00007efdb03db350> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb03d6648> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 waiting for monitor entry [0x00007efbe7175000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:305) at kafka.utils.Logging$class.info(Logging.scala:66) at kafka.utils.ShutdownableThread.info(ShutdownableThread.scala:24) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:224) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae889128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f runnable [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at kafka.log.LogSegment.append(LogSegment.scala:139) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a4e3950> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaea48250> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1c1118> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e runnable [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdb0154fc8> (a sun.nio.ch.Util$3) - locked <0x00007efdb0154fb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdb01502a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 runnable [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f runnable [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at java.lang.AbstractStringBuilder.(AbstractStringBuilder.java:68) at java.lang.StringBuilder.(StringBuilder.java:89) at org.apache.kafka.common.utils.Sanitizer.sanitize(Sanitizer.java:56) at kafka.network.RequestChannel$Session.(RequestChannel.scala:52) at kafka.network.RequestChannel$Request.(RequestChannel.scala:88) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:696) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 waiting for monitor entry [0x00007efbff6f5000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.log4j.Category.callAppenders(Category.java:204) - waiting to lock <0x00007efd500d4d20> (a org.apache.log4j.spi.RootLogger) at org.apache.log4j.Category.forcedLog(Category.java:391) at org.apache.log4j.Category.log(Category.java:856) at org.slf4j.impl.Log4jLoggerAdapter.info(Log4jLoggerAdapter.java:347) at org.apache.kafka.common.security.authenticator.SaslServerCallbackHandler.handleAuthorizeCallback(SaslServerCallbackHandler.java:70) at org.apache.kafka.common.security.authenticator.SaslServerCallbackHandler.handle(SaslServerCallbackHandler.java:56) at com.sun.security.sasl.gsskerb.GssKrb5Server.doHandshake2(GssKrb5Server.java:317) at com.sun.security.sasl.gsskerb.GssKrb5Server.evaluateResponse(GssKrb5Server.java:161) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.handleSaslToken(SaslServerAuthenticator.java:377) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.authenticate(SaslServerAuthenticator.java:261) at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:127) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:487) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at kafka.network.Processor.processCompletedSends(SocketServer.scala:715) at kafka.network.Processor.run(SocketServer.scala:585) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:120) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 waiting on condition [0x00007efc054d1000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at java.util.regex.Matcher.(Matcher.java:225) at java.util.regex.Pattern.matcher(Pattern.java:1093) at org.apache.kafka.common.security.kerberos.KerberosRule.apply(KerberosRule.java:181) at org.apache.kafka.common.security.kerberos.KerberosShortNamer.shortName(KerberosShortNamer.java:93) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.applyKerberosShortNamer(DefaultKafkaPrincipalBuilder.java:131) at org.apache.kafka.common.security.authenticator.DefaultKafkaPrincipalBuilder.build(DefaultKafkaPrincipalBuilder.java:120) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.principal(SaslServerAuthenticator.java:286) at org.apache.kafka.common.network.KafkaChannel.principal(KafkaChannel.java:114) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:695) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 waiting on condition [0x00007efc05d3f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.network.NetworkReceive.(NetworkReceive.java:71) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:293) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 waiting on condition [0x00007efc06e5b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 waiting on condition [0x00007efc06f5c000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: RUNNABLE at java.util.LinkedList.listIterator(LinkedList.java:868) at javax.security.auth.Subject$SecureSet$1.(Subject.java:1050) at javax.security.auth.Subject$SecureSet.iterator(Subject.java:1049) at java.util.Collections$SynchronizedCollection.iterator(Collections.java:2031) at javax.security.auth.Subject$ClassSet.populateSet(Subject.java:1382) at javax.security.auth.Subject$ClassSet.(Subject.java:1366) - locked <0x00007f04e881c678> (a java.util.Collections$SynchronizedSet) at javax.security.auth.Subject.getPrincipals(Subject.java:624) at org.apache.hadoop.security.UserGroupInformation.(UserGroupInformation.java:546) at org.apache.hadoop.security.UserGroupInformation.createRemoteUser(UserGroupInformation.java:1273) at org.apache.hadoop.security.UserGroupInformation.createRemoteUser(UserGroupInformation.java:1256) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:580) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007efe36b0d340> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.requests.FetchResponse.addPartitionData(FetchResponse.java:456) at org.apache.kafka.common.requests.FetchResponse.addTopicData(FetchResponse.java:448) at org.apache.kafka.common.requests.FetchResponse.addResponseData(FetchResponse.java:433) at org.apache.kafka.common.requests.FetchResponse.toSend(FetchResponse.java:375) at org.apache.kafka.common.requests.RequestContext.buildResponse(RequestContext.java:78) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2350) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 waiting on condition [0x00007ef533c35000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91012370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 waiting on condition [0x00007ef533f12000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: RUNNABLE at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1577) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007effc867cf70> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: RUNNABLE at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:188) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:405) at kafka.server.IncrementalFetchContext$$anonfun$foreachPartition$3.apply(FetchSession.scala:404) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at kafka.server.IncrementalFetchContext.foreachPartition(FetchSession.scala:404) - locked <0x00007efdfc01f3d0> (a kafka.server.FetchSession) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 runnable [0x00007efb1c231000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efe7a2f6500> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting for monitor entry [0x00007efb244b4000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.logSegments(Log.scala:1711) - waiting to lock <0x00007efe78024920> (a java.lang.Object) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd508677c8> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b runnable [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a runnable [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 runnable [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.locks.AbstractQueuedSynchronizer.addWaiter(AbstractQueuedSynchronizer.java:606) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:1199) at java.util.concurrent.locks.ReentrantLock$NonfairSync.lock(ReentrantLock.java:209) at java.util.concurrent.locks.ReentrantLock.lock(ReentrantLock.java:285) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.offer(ScheduledThreadPoolExecutor.java:1010) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.add(ScheduledThreadPoolExecutor.java:1037) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.add(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ScheduledThreadPoolExecutor.reExecutePeriodic(ScheduledThreadPoolExecutor.java:346) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:296) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50058e70> (a java.util.concurrent.ThreadPoolExecutor$Worker) "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.ConcurrentHashMap$Traverser.advance(ConcurrentHashMap.java:3339) at java.util.concurrent.ConcurrentHashMap$EntryIterator.next(ConcurrentHashMap.java:3460) at java.util.concurrent.ConcurrentHashMap$EntryIterator.next(ConcurrentHashMap.java:3446) at java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet$1.next(Collections.java:1661) at java.util.Collections$UnmodifiableMap$UnmodifiableEntrySet$1.next(Collections.java:1654) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:311) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:16:54 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #945 daemon prio=10 os_prio=0 tid=0x00007ef67c014000 nid=0x21c0 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe2c9cc3c8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe789b3ae8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c932810> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf20000e0> (a sun.nio.ch.Util$3) - locked <0x00007efdf20000f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2000068> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c waiting on condition [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd59eedea0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91460f00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6a79c68> (a sun.nio.ch.Util$3) - locked <0x00007efeb6a79c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6a79bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe7dc7e1b8> (a java.lang.Object) - locked <0x00007efe7dc7e198> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 waiting on condition [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 waiting on condition [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe6429eef8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0f3a1f80> (a java.lang.Object) - locked <0x00007efe0f3a1f60> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff7a500e68> (a java.lang.Object) - locked <0x00007eff7a500e48> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2358778> (a sun.nio.ch.Util$3) - locked <0x00007efdf2358790> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2358700> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 waiting on condition [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe72516fb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 waiting on condition [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efeb5bad3d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80cfbaf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe2989b1f0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8c642970> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c50eef0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe64f9a818> (a java.lang.Object) - locked <0x00007efe64f9a7f8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe72b2fc08> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efeb5ba08c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b80f918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c waiting on condition [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7244b250> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b waiting on condition [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe723e5348> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad779da8> (a sun.nio.ch.Util$3) - locked <0x00007efdad779d90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad774ff0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe2988e2f0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efeb4475750> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdcc0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 waiting on condition [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b940d58> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe78260cd0> (a java.lang.Object) - locked <0x00007efe78260ce0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2475f00> (a sun.nio.ch.Util$3) - locked <0x00007efdf2475f18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2475e88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6cb4b78> (a sun.nio.ch.Util$3) - locked <0x00007efeb6cb4b90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6cb4b00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe295da468> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe72a74250> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4e0908> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efed0f614d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c2488d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 waiting on condition [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a072c78> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bb2b880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 waiting on condition [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7244b290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff waiting for monitor entry [0x00007ef7e52fd000] java.lang.Thread.State: BLOCKED (on object monitor) at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe782eb438> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910cdaa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd waiting on condition [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6a79e28> (a sun.nio.ch.Util$3) - locked <0x00007efeb6a79e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6a79db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb waiting on condition [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7808df68> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b17ec68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efed4b5cf70> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7ad12768> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b3a0648> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6dd2300> (a sun.nio.ch.Util$3) - locked <0x00007efeb6dd2318> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6dd2288> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 waiting on condition [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7801d220> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7f6b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee waiting on condition [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdad314350> (a java.lang.Object) - locked <0x00007efdad314330> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 waiting on condition [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdadb30438> (a java.lang.Object) - locked <0x00007efdadb30418> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 waiting on condition [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe724b0f00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d0041e0> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0041f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004168> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 waiting on condition [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 waiting on condition [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c930078> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf waiting on condition [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb4c56e70> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd59c9d1d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91012370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb7f33838> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8061adb0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839c8628> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe9144bd00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 waiting on condition [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf23588c0> (a sun.nio.ch.Util$3) - locked <0x00007efdf23588d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd65178> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b36cf30> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d583b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88d583a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d3b918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 waiting on condition [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdf223af10> (a java.lang.Object) - locked <0x00007efdf223af20> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff7ae53318> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839bff90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdaf43a960> (a java.lang.Object) - locked <0x00007efdaf43a940> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c2652b8> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2652a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c264fa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe832e4d78> (a java.lang.Object) - locked <0x00007efe832e4d58> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c230740> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6dd24c0> (a sun.nio.ch.Util$3) - locked <0x00007efeb6dd24d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6dd2448> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 waiting on condition [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf waiting on condition [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.map0(Native Method) at sun.nio.ch.FileChannelImpl.map(FileChannelImpl.java:924) - locked <0x00007f04eb100278> (a java.lang.Object) at kafka.log.AbstractIndex$$anonfun$resize$1.apply$mcZ$sp(AbstractIndex.scala:130) at kafka.log.AbstractIndex$$anonfun$resize$1.apply(AbstractIndex.scala:115) at kafka.log.AbstractIndex$$anonfun$resize$1.apply(AbstractIndex.scala:115) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.AbstractIndex.resize(AbstractIndex.scala:115) at kafka.log.TimeIndex.kafka$log$TimeIndex$$super$resize(TimeIndex.scala:189) at kafka.log.TimeIndex$$anonfun$resize$1.apply$mcZ$sp(TimeIndex.scala:189) at kafka.log.TimeIndex$$anonfun$resize$1.apply(TimeIndex.scala:189) at kafka.log.TimeIndex$$anonfun$resize$1.apply(TimeIndex.scala:189) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.TimeIndex.resize(TimeIndex.scala:188) at kafka.log.AbstractIndex$$anonfun$trimToValidSize$1.apply$mcZ$sp(AbstractIndex.scala:184) at kafka.log.AbstractIndex$$anonfun$trimToValidSize$1.apply(AbstractIndex.scala:184) at kafka.log.AbstractIndex$$anonfun$trimToValidSize$1.apply(AbstractIndex.scala:184) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.AbstractIndex.trimToValidSize(AbstractIndex.scala:183) at kafka.log.LogSegment.onBecomeInactiveSegment(LogSegment.scala:502) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$30.apply(Log.scala:1481) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$30.apply(Log.scala:1481) at scala.Option.foreach(Option.scala:257) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1481) - locked <0x00007efe7a081348> (a java.lang.Object) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1467) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.roll(Log.scala:1467) at kafka.log.Log.kafka$log$Log$$maybeRoll(Log.scala:1452) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:859) - locked <0x00007efe7a081348> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) - <0x00007eff34fde348> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd waiting on condition [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb waiting on condition [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd59955328> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf2d98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981ba8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981bc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdad897bb8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf0201b8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf0201a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf01b498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe646f4600> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78010540> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b927ba8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 waiting on condition [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 waiting on condition [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0f131e70> (a java.lang.Object) - locked <0x00007efe0f131e50> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd81ce8> (a sun.nio.ch.Util$3) - locked <0x00007efdacd81cd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd7cfc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf waiting on condition [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6dd2680> (a sun.nio.ch.Util$3) - locked <0x00007efeb6dd2698> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6dd2608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2476168> (a sun.nio.ch.Util$3) - locked <0x00007efdf2476180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf24760f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8b98b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8b9898> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8b4b90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdad8a5330> (a java.lang.Object) - locked <0x00007efdad8a5310> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf223b170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c116f28> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88cac830> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 waiting on condition [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0fd10698> (a java.lang.Object) - locked <0x00007efe0fd10678> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e waiting on condition [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf211d868> (a sun.nio.ch.Util$3) - locked <0x00007efdf211d880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf211d7f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe72e4d700> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe72dd0eb8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe7869daf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2000340> (a sun.nio.ch.Util$3) - locked <0x00007efdf2000358> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf20002c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae01f8f8> (a sun.nio.ch.Util$3) - locked <0x00007efdae01f8e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae01bf60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe83538240> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b7e5c98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 waiting on condition [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2358ae8> (a sun.nio.ch.Util$3) - locked <0x00007efdf2358b00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2358a70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdac17c0b0> (a java.lang.Object) - locked <0x00007efdac17c090> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdad84ecb8> (a java.lang.Object) - locked <0x00007efdad84ec98> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe29c133d0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd58fbc920> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8bcf0160> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0fc04958> (a java.lang.Object) - locked <0x00007efe0fc04938> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 waiting on condition [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaf98c380> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 waiting on condition [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff28f22dd8> (a java.lang.Object) - locked <0x00007eff28f22db8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6cb4d38> (a sun.nio.ch.Util$3) - locked <0x00007efeb6cb4d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6cb4cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a5562a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d05b90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f waiting on condition [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdaf0900c8> (a java.lang.Object) - locked <0x00007efdaf0900a8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf211da28> (a sun.nio.ch.Util$3) - locked <0x00007efdf211da40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf211d9b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0fce98e0> (a java.lang.Object) - locked <0x00007efe0fce98c0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdad04de38> (a java.lang.Object) - locked <0x00007efdad04de18> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 waiting for monitor entry [0x00007ef952de6000] java.lang.Thread.State: BLOCKED (on object monitor) at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912a0388> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 waiting on condition [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0818e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae0818c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae07df48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdf211db08> (a java.lang.Object) - locked <0x00007efdf211db18> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 waiting on condition [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efeb6b97360> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e waiting on condition [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe643e1770> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe72da6638> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efda9cdf438> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6dd29f8> (a sun.nio.ch.Util$3) - locked <0x00007efeb6dd2a10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6dd2980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 waiting on condition [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf25935a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf223b2e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d waiting on condition [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf19d4d8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf19d4c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1987b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 waiting on condition [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6b97480> (a sun.nio.ch.Util$3) - locked <0x00007efeb6b97498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6b97408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at scala.collection.immutable.MapLike$class.keySet(MapLike.scala:107) at scala.collection.immutable.AbstractMap.keySet(Map.scala:194) at scala.collection.immutable.AbstractMap.keySet(Map.scala:194) at scala.collection.MapLike$class.keys(MapLike.scala:192) at scala.collection.AbstractMap.keys(Map.scala:59) at kafka.server.ReplicaFetcherThread.buildLeaderEpochRequest(ReplicaFetcherThread.scala:354) at kafka.server.AbstractFetcherThread$$anonfun$3.apply(AbstractFetcherThread.scala:127) at kafka.server.AbstractFetcherThread$$anonfun$3.apply(AbstractFetcherThread.scala:127) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.maybeTruncate(AbstractFetcherThread.scala:127) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:103) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdf33752e8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 waiting on condition [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5bf5f598> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1bc5d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe780154f0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe915251f0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7efa28> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7efa10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7ead98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51) at sun.nio.ch.IOUtil.write(IOUtil.java:148) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:504) - locked <0x00007efe8cb2e848> (a java.lang.Object) at java.nio.channels.SocketChannel.write(SocketChannel.java:502) at org.apache.kafka.common.network.PlaintextTransportLayer.write(PlaintextTransportLayer.java:153) at org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:60) at org.apache.kafka.common.network.KafkaChannel.send(KafkaChannel.java:339) at org.apache.kafka.common.network.KafkaChannel.write(KafkaChannel.java:310) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:512) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf211de20> (a sun.nio.ch.Util$3) - locked <0x00007efdf211de38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf211dda8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b waiting on condition [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a waiting on condition [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad52dc58> (a sun.nio.ch.Util$3) - locked <0x00007efdad52dc40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad528ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf20005c8> (a sun.nio.ch.Util$3) - locked <0x00007efdf20005e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2000550> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe648bd768> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe2cc88418> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c62c958> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efeb6a740a0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe80db4e00> (a java.lang.Object) - locked <0x00007efe80db4de0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6cb4ef8> (a sun.nio.ch.Util$3) - locked <0x00007efeb6cb4f10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6cb4e80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf27cf1c0> (a sun.nio.ch.Util$3) - locked <0x00007efdf27cf1d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf27cf148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe837f0208> (a sun.nio.ch.Util$3) - locked <0x00007efe837f01f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe837ece18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 waiting on condition [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafad2c98> (a sun.nio.ch.Util$3) - locked <0x00007efdafad2c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafacdfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d waiting on condition [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.network.Selector.poll(Selector.java:389) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8caa4ab8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf181c70> (a sun.nio.ch.Util$3) - locked <0x00007efdaf181c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf17cf50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe91479b28> (a java.lang.Object) - locked <0x00007efe91479b08> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe91511158> (a java.lang.Object) - locked <0x00007efe91511138> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007f04ec300688> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7aa25aa0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c17fa60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe7d050118> (a java.lang.Object) - locked <0x00007efe7d0500f8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe2cae3ee8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78aa0d90> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdf24764f0> (a java.lang.Object) - locked <0x00007efdf2476500> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e waiting on condition [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf2358bc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdf348e940> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe64d4e5a0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe782f4900> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe910b4128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2000788> (a sun.nio.ch.Util$3) - locked <0x00007efdf20007a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2000710> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe785e29b0> (a java.lang.Object) - locked <0x00007efe785e2990> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdaf89a1c0> (a java.lang.Object) - locked <0x00007efdaf89a1a0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c19caf0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0f0c7100> (a java.lang.Object) - locked <0x00007efe0f0c70e0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 waiting on condition [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 waiting on condition [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0fab3838> (a java.lang.Object) - locked <0x00007efe0fab3818> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe91e27840> (a java.lang.Object) - locked <0x00007efe91e27820> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdae6cd0e8> (a java.lang.Object) - locked <0x00007efdae6cd0c8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb waiting on condition [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007eff13b1b700> (a java.lang.Object) - locked <0x00007eff13b1b6e0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe91df26b0> (a java.lang.Object) - locked <0x00007efe91df2690> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at java.io.FileOutputStream.open0(Native Method) at java.io.FileOutputStream.open(FileOutputStream.java:270) at java.io.FileOutputStream.(FileOutputStream.java:213) at java.io.FileOutputStream.(FileOutputStream.java:162) at kafka.log.ProducerStateManager$.kafka$log$ProducerStateManager$$writeSnapshot(ProducerStateManager.scala:433) at kafka.log.ProducerStateManager.takeSnapshot(ProducerStateManager.scala:666) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1489) - locked <0x00007efe782e7db0> (a java.lang.Object) at kafka.log.Log$$anonfun$roll$2.apply(Log.scala:1467) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.roll(Log.scala:1467) at kafka.log.Log.kafka$log$Log$$maybeRoll(Log.scala:1452) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:859) - locked <0x00007efe782e7db0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80bfbc98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdabf717b0> (a java.lang.Object) - locked <0x00007efdabf71790> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd506b3328> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdf3a86d58> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe641b4568> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe8c332270> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe64c15980> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe64033fa8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78aa3470> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeb51428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdada3db40> (a java.lang.Object) - locked <0x00007efdada3db20> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe4dc41178> (a java.lang.Object) - locked <0x00007efe4dc41158> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdafe31460> (a java.lang.Object) - locked <0x00007efdafe31440> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 waiting on condition [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efffdfb3170> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd506fd370> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d724590> (a sun.nio.ch.Util$3) - locked <0x00007efe7d7245a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d724518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 waiting on condition [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 waiting for monitor entry [0x00007efad9225000] java.lang.Thread.State: BLOCKED (on object monitor) at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae51d880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Crc32C$PureJavaChecksumFactory.create(Crc32C.java:111) at org.apache.kafka.common.utils.Crc32C.create(Crc32C.java:77) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:71) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad9988d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efeb5b963c8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdf3a693e8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0fc07718> (a java.lang.Object) - locked <0x00007efe0fc076f8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdf20f03d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da waiting on condition [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c295188> (a sun.nio.ch.Util$3) - locked <0x00007efe8c2951a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c295110> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 waiting on condition [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf211df00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce waiting on condition [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdae949c88> (a java.lang.Object) - locked <0x00007efdae949c68> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf323b8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdf323b8d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf323b840> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 waiting on condition [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf223b4f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.writev0(Native Method) at sun.nio.ch.SocketDispatcher.writev(SocketDispatcher.java:51) at sun.nio.ch.IOUtil.write(IOUtil.java:148) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:504) - locked <0x00007efdad979a68> (a java.lang.Object) at java.nio.channels.SocketChannel.write(SocketChannel.java:502) at org.apache.kafka.common.network.PlaintextTransportLayer.write(PlaintextTransportLayer.java:153) at org.apache.kafka.common.network.ByteBufferSend.writeTo(ByteBufferSend.java:60) at org.apache.kafka.common.network.KafkaChannel.send(KafkaChannel.java:339) at org.apache.kafka.common.network.KafkaChannel.write(KafkaChannel.java:310) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:512) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 waiting on condition [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdaf87f5e0> (a java.lang.Object) - locked <0x00007efdaf87f5c0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdad86f230> (a java.lang.Object) - locked <0x00007efdad86f210> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf25936c8> (a sun.nio.ch.Util$3) - locked <0x00007efdf25936e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2593650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2e7fd28> (a sun.nio.ch.Util$3) - locked <0x00007efdf2e7fd40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2e7fcb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0f1ab240> (a java.lang.Object) - locked <0x00007efe0f1ab220> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at org.apache.kafka.common.record.MemoryRecords.validBytes(MemoryRecords.java:110) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:188) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad50e370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe64978640> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78022b10> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c230c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2593870> (a sun.nio.ch.Util$3) - locked <0x00007efdf36d22e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf36d1fd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa waiting on condition [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0f983910> (a java.lang.Object) - locked <0x00007efe0f9838f0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdad24ef00> (a java.lang.Object) - locked <0x00007efdad24eee0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 waiting on condition [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at java.util.HashMap.newNode(HashMap.java:1750) at java.util.HashMap.putVal(HashMap.java:631) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at sun.nio.ch.EPollSelectorImpl.updateSelectedKeys(EPollSelectorImpl.java:131) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:98) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe641b0c50> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe80100cc0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe72dd86c0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf323bac8> (a sun.nio.ch.Util$3) - locked <0x00007efdf323bae0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f waiting on condition [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac7f3120> (a sun.nio.ch.Util$3) - locked <0x00007efdac7f3108> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac7ee440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7a52b2a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c86e598> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad624c40> (a sun.nio.ch.Util$3) - locked <0x00007efdad624c28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad61ff60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 waiting on condition [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac702000> (a sun.nio.ch.Util$3) - locked <0x00007efdac701fe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab927170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe0fab5010> (a java.lang.Object) - locked <0x00007efe0fab4ff0> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2e7ffb0> (a sun.nio.ch.Util$3) - locked <0x00007efdf2e7ffc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2e7ff38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efdf2d62128> (a java.lang.Object) - locked <0x00007efdf2d62138> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf323bbd8> (a sun.nio.ch.Util$3) - locked <0x00007efdf323bbf0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf323bb60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe7299cb00> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe7a2c63d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae889128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f waiting on condition [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efeb529c2e8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaea48250> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc1e550> (a sun.nio.ch.Util$3) - locked <0x00007efdafc1e538> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc19830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e waiting on condition [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe34a33940> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe800da278> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaee430d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at java.lang.Long.valueOf(Long.java:840) at org.apache.kafka.common.network.Selector$IdleExpiryManager.update(Selector.java:1081) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:464) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.requests.FetchRequest.toStruct(FetchRequest.java:473) at org.apache.kafka.common.requests.AbstractRequest.serialize(AbstractRequest.java:98) at org.apache.kafka.common.requests.AbstractRequest.toSend(AbstractRequest.java:91) at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:476) at org.apache.kafka.clients.NetworkClient.doSend(NetworkClient.java:450) at org.apache.kafka.clients.NetworkClient.send(NetworkClient.java:411) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:91) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 waiting on condition [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f waiting on condition [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b runnable [0x00007efbffdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d128> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d110> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d140> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollCtl(Native Method) at sun.nio.ch.EPollArrayWrapper.updateRegistrations(EPollArrayWrapper.java:299) - locked <0x00007efd832eaf50> (a java.lang.Object) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:268) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d578> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d560> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d590> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682428> (a sun.nio.ch.Util$3) - locked <0x00007efd76682410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 runnable [0x00007efc05a3c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678d9c8> (a sun.nio.ch.Util$3) - locked <0x00007efd7678d9b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678d9e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685ea70> (a sun.nio.ch.Util$3) - locked <0x00007efd7685ea58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685ea88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 waiting on condition [0x00007efc05c3e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 waiting on condition [0x00007efc05d3f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 waiting on condition [0x00007efc05e40000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c runnable [0x00007efc06542000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: RUNNABLE at kafka.server.FetchSessionCache.get(FetchSession.scala:567) - locked <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.FetchManager.newContext(FetchSession.scala:760) - locked <0x00007efd80a6e438> (a kafka.server.FetchSessionCache) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:506) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: RUNNABLE at java.util.ArrayList.iterator(ArrayList.java:840) at scala.collection.convert.Wrappers$JListWrapper.iterator(Wrappers.scala:88) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1$$anonfun$apply$1.apply(MetadataCache.scala:94) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1$$anonfun$apply$1.apply(MetadataCache.scala:73) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1.apply(MetadataCache.scala:73) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1.apply(MetadataCache.scala:72) at scala.Option.map(Option.scala:146) at kafka.server.MetadataCache.kafka$server$MetadataCache$$getPartitionMetadata(MetadataCache.scala:72) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1$$anonfun$apply$7.apply(MetadataCache.scala:129) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1$$anonfun$apply$7.apply(MetadataCache.scala:128) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:241) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:104) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1.apply(MetadataCache.scala:128) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1.apply(MetadataCache.scala:128) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.MetadataCache.getTopicMetadata(MetadataCache.scala:127) at kafka.server.KafkaApis.getTopicMetadata(KafkaApis.scala:998) at kafka.server.KafkaApis.handleTopicMetadataRequest(KafkaApis.scala:1082) at kafka.server.KafkaApis.handle(KafkaApis.scala:109) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 runnable [0x00007efc09208000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: RUNNABLE at kafka.server.MetadataCache.kafka$server$MetadataCache$$getEndpoints(MetadataCache.scala:56) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1$$anonfun$apply$1.apply(MetadataCache.scala:78) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1$$anonfun$apply$1.apply(MetadataCache.scala:73) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) at scala.collection.AbstractTraversable.map(Traversable.scala:104) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1.apply(MetadataCache.scala:73) at kafka.server.MetadataCache$$anonfun$kafka$server$MetadataCache$$getPartitionMetadata$1.apply(MetadataCache.scala:72) at scala.Option.map(Option.scala:146) at kafka.server.MetadataCache.kafka$server$MetadataCache$$getPartitionMetadata(MetadataCache.scala:72) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1$$anonfun$apply$7.apply(MetadataCache.scala:129) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1$$anonfun$apply$7.apply(MetadataCache.scala:128) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:241) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:104) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1.apply(MetadataCache.scala:128) at kafka.server.MetadataCache$$anonfun$getTopicMetadata$1.apply(MetadataCache.scala:128) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.MetadataCache.getTopicMetadata(MetadataCache.scala:127) at kafka.server.KafkaApis.getTopicMetadata(KafkaApis.scala:998) at kafka.server.KafkaApis.handleTopicMetadataRequest(KafkaApis.scala:1082) at kafka.server.KafkaApis.handle(KafkaApis.scala:109) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 waiting on condition [0x00007ef6aa1ba000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f runnable [0x00007ef607476000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 waiting on condition [0x00007ef5383b3000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b80f918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.partitionCount(AbstractFetcherThread.scala:392) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:159) at kafka.server.AbstractFetcherManager$$anonfun$shutdownIdleFetcherThreads$2.apply(AbstractFetcherManager.scala:158) at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:733) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashMap$$anonfun$foreach$1.apply(HashMap.scala:130) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap.foreach(HashMap.scala:130) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.server.AbstractFetcherManager.shutdownIdleFetcherThreads(AbstractFetcherManager.scala:158) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1130) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 runnable [0x00007ef53851b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 waiting on condition [0x00007efaf114e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireQueued(AbstractQueuedSynchronizer.java:870) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2083) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.getObjectVolatile(Native Method) at java.util.concurrent.atomic.AtomicReferenceArray.getRaw(AtomicReferenceArray.java:130) at java.util.concurrent.atomic.AtomicReferenceArray.get(AtomicReferenceArray.java:125) at com.google.common.cache.LocalCache$Segment.getFirst(LocalCache.java:2730) at com.google.common.cache.LocalCache$Segment.getEntry(LocalCache.java:2737) at com.google.common.cache.LocalCache$Segment.getLiveEntry(LocalCache.java:2758) at com.google.common.cache.LocalCache$Segment.containsKey(LocalCache.java:2817) at com.google.common.cache.LocalCache.containsKey(LocalCache.java:4216) at java.util.Collections$SetFromMap.contains(Collections.java:5459) at org.apache.hadoop.security.Groups.getGroups(Groups.java:222) at org.apache.hadoop.security.UserGroupInformation.getGroups(UserGroupInformation.java:1588) at org.apache.hadoop.security.UserGroupInformation.getGroupNames(UserGroupInformation.java:1576) at org.apache.ranger.audit.provider.MiscUtil.getGroupsForRequestUser(MiscUtil.java:581) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:163) at org.apache.ranger.authorization.kafka.authorizer.RangerKafkaAuthorizer.authorize(RangerKafkaAuthorizer.java:135) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$kafka$server$KafkaApis$$authorize$1.apply(KafkaApis.scala:378) at scala.Option.forall(Option.scala:247) at kafka.server.KafkaApis.kafka$server$KafkaApis$$authorize(KafkaApis.scala:378) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:535) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$3.apply(KafkaApis.scala:534) at kafka.server.SessionlessFetchContext$$anonfun$foreachPartition$1.apply(FetchSession.scala:331) at kafka.server.SessionlessFetchContext$$anonfun$foreachPartition$1.apply(FetchSession.scala:331) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.server.SessionlessFetchContext.foreachPartition(FetchSession.scala:331) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:534) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 waiting on condition [0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f runnable [0x00007efc08705000] java.lang.Thread.State: RUNNABLE at java.io.FileInputStream.close0(Native Method) at java.io.FileInputStream.access$000(FileInputStream.java:49) at java.io.FileInputStream$1.close(FileInputStream.java:336) at java.io.FileDescriptor.closeAll(FileDescriptor.java:212) - locked <0x00007f054f01c2a0> (a java.io.FileDescriptor) at java.io.FileInputStream.close(FileInputStream.java:334) at sun.nio.cs.StreamDecoder.implClose(StreamDecoder.java:378) at sun.nio.cs.StreamDecoder.close(StreamDecoder.java:193) - locked <0x00007f054f01c1f0> (a java.io.InputStreamReader) at java.io.InputStreamReader.close(InputStreamReader.java:199) at java.io.BufferedReader.close(BufferedReader.java:525) - locked <0x00007f054f01c1f0> (a java.io.InputStreamReader) at kafka.server.checkpoints.CheckpointFile.liftedTree2$1(CheckpointFile.scala:119) at kafka.server.checkpoints.CheckpointFile.read(CheckpointFile.scala:86) - locked <0x00007efd7696f510> (a java.lang.Object) at kafka.server.checkpoints.OffsetCheckpointFile.read(OffsetCheckpointFile.scala:61) at kafka.log.LogCleanerManager$$anonfun$allCleanerCheckpoints$1$$anonfun$apply$1.apply(LogCleanerManager.scala:89) at kafka.log.LogCleanerManager$$anonfun$allCleanerCheckpoints$1$$anonfun$apply$1.apply(LogCleanerManager.scala:87) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.MapLike$DefaultValuesIterable.foreach(MapLike.scala:206) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:241) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:104) at kafka.log.LogCleanerManager$$anonfun$allCleanerCheckpoints$1.apply(LogCleanerManager.scala:87) at kafka.log.LogCleanerManager$$anonfun$allCleanerCheckpoints$1.apply(LogCleanerManager.scala:95) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.LogCleanerManager.allCleanerCheckpoints(LogCleanerManager.scala:86) at kafka.log.LogCleanerManager$$anonfun$grabFilthiestCompactedLog$1.apply(LogCleanerManager.scala:126) at kafka.log.LogCleanerManager$$anonfun$grabFilthiestCompactedLog$1.apply(LogCleanerManager.scala:123) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.log.LogCleanerManager.grabFilthiestCompactedLog(LogCleanerManager.scala:123) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:296) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd7696f620> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting for monitor entry [0x00007efb00172000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.logSegments(Log.scala:1711) - waiting to lock <0x00007efe7aa25aa0> (a java.lang.Object) at kafka.log.Log$$anonfun$flush$1.apply$mcV$sp(Log.scala:1536) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log$$anonfun$flush$1.apply(Log.scala:1531) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.flush(Log.scala:1531) at kafka.log.Log$$anonfun$roll$2$$anonfun$apply$1.apply$mcV$sp(Log.scala:1506) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506d7910> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efe7aa25aa0> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 runnable [0x00007efb1d7fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b waiting on condition [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a waiting on condition [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 waiting on condition [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 waiting on condition [0x00007efc63efd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 runnable [0x00007efc63ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.ConcurrentSkipListMap.findPredecessor(ConcurrentSkipListMap.java:691) at java.util.concurrent.ConcurrentSkipListMap.doGet(ConcurrentSkipListMap.java:781) at java.util.concurrent.ConcurrentSkipListMap.get(ConcurrentSkipListMap.java:1546) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricHolder.evict(TimelineMetricsCache.java:118) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache.getTimelineMetric(TimelineMetricsCache.java:175) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.populateMetricsList(KafkaTimelineMetricsReporter.java:487) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:419) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processGauge(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Gauge.processWith(Gauge.java:28) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ========== 2020-12-28 13:17:05 Full thread dump Java HotSpot(TM) 64-Bit Server VM (25.212-b10 mixed mode): "process reaper" #945 daemon prio=10 os_prio=0 tid=0x00007ef67c014000 nid=0x21c0 waiting on condition [0x00007efc62493000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678f770> (a java.util.concurrent.SynchronousQueue$TransferStack) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:460) at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362) at java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:941) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1073) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Attach Listener" #914 daemon prio=9 os_prio=0 tid=0x00007efcac009000 nid=0x1c5a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "executor-Fetch" #886 prio=5 os_prio=0 tid=0x00007ef3f8008000 nid=0x1905 waiting on condition [0x00007ef61990d000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd86a20c98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1019" #690 prio=5 os_prio=0 tid=0x00007ef65ed19000 nid=0x8b41 runnable [0x00007ef651973000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf80b860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf80b848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf806b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1061" #689 prio=5 os_prio=0 tid=0x00007ef65ecfe000 nid=0x8b40 runnable [0x00007ef651d14000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb5f8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb5f8a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb5ab98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1014" #688 prio=5 os_prio=0 tid=0x00007ef65ece3000 nid=0x8b3f runnable [0x00007ef654477000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf002d68> (a sun.nio.ch.Util$3) - locked <0x00007efdaf002d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4b1da8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1064" #687 prio=5 os_prio=0 tid=0x00007ef65ecc8800 nid=0x8b3e runnable [0x00007ef655db1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf20000e0> (a sun.nio.ch.Util$3) - locked <0x00007efdf20000f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2000068> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1009" #686 prio=5 os_prio=0 tid=0x00007ef65ecad800 nid=0x8b3d runnable [0x00007ef655eb2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd12108> (a sun.nio.ch.Util$3) - locked <0x00007efdafd120f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd0d3e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1051" #685 prio=5 os_prio=0 tid=0x00007ef65ec93000 nid=0x8b3c waiting on condition [0x00007ef656bfc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe782606d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1010" #684 prio=5 os_prio=0 tid=0x00007ef65ec78000 nid=0x8b3b runnable [0x00007ef6621fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf33603a0> (a sun.nio.ch.Util$3) - locked <0x00007efdf3360388> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf3360090> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1036" #683 prio=5 os_prio=0 tid=0x00007ef65ec5d000 nid=0x8b3a runnable [0x00007ef6882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6a79c68> (a sun.nio.ch.Util$3) - locked <0x00007efeb6a79c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6a79bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1004" #682 prio=5 os_prio=0 tid=0x00007ef65ec42800 nid=0x8b39 runnable [0x00007ef6883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d8b6ef8> (a sun.nio.ch.Util$3) - locked <0x00007efe7d8b6ee0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d8b6be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1054" #681 prio=5 os_prio=0 tid=0x00007ef65ec27800 nid=0x8b38 runnable [0x00007ef688aa6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe784c3e08> (a sun.nio.ch.Util$3) - locked <0x00007efe784c3e20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe784c3d90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1024" #680 prio=5 os_prio=0 tid=0x00007ef65ec0c800 nid=0x8b37 waiting on condition [0x00007ef6981fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78981740> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1020" #679 prio=5 os_prio=0 tid=0x00007ef65ebf2000 nid=0x8b36 runnable [0x00007ef6982fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc036f8> (a sun.nio.ch.Util$3) - locked <0x00007efdafc036e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbfe9d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1031" #678 prio=5 os_prio=0 tid=0x00007ef65ebd7000 nid=0x8b35 waiting on condition [0x00007ef6a8156000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe6429eef8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1020" #677 prio=5 os_prio=0 tid=0x00007ef65ebbc800 nid=0x8b34 runnable [0x00007ef6a8257000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac583ce0> (a sun.nio.ch.Util$3) - locked <0x00007efdac583cc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac57efc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1049" #676 prio=5 os_prio=0 tid=0x00007ef65eba1800 nid=0x8b33 runnable [0x00007ef6ac210000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98db358> (a sun.nio.ch.Util$3) - locked <0x00007efda98db340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98db2c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1065" #675 prio=5 os_prio=0 tid=0x00007ef65eb87000 nid=0x8b32 runnable [0x00007ef6ac311000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae1d4950> (a sun.nio.ch.Util$3) - locked <0x00007efdae1d4938> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae1cfb98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1022" #674 prio=5 os_prio=0 tid=0x00007ef65eb6c000 nid=0x8b31 runnable [0x00007ef6b1ebb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadc126a8> (a sun.nio.ch.Util$3) - locked <0x00007efdadc12690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadc0d8f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1018" #673 prio=5 os_prio=0 tid=0x00007ef65eb51000 nid=0x8b30 runnable [0x00007ef6b8eaa000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4f3790> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1026" #672 prio=5 os_prio=0 tid=0x00007ef65eb36000 nid=0x8b2f runnable [0x00007ef6d1bd1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b68a8> (a sun.nio.ch.Util$3) - locked <0x00007efda97b6890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b6818> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1015" #671 prio=5 os_prio=0 tid=0x00007ef65eb1b800 nid=0x8b2e runnable [0x00007ef6de12f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007eff31117190> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efeb5ba2090> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d32a38> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1046" #670 prio=5 os_prio=0 tid=0x00007ef65eb00800 nid=0x8b2d runnable [0x00007ef6e0ae8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe782608a0> (a sun.nio.ch.Util$3) - locked <0x00007efe782608b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78260828> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1054" #669 prio=5 os_prio=0 tid=0x00007ef65eae5800 nid=0x8b2c runnable [0x00007ef6e5d2f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe787246f8> (a sun.nio.ch.Util$3) - locked <0x00007efe78724710> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724680> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1063" #668 prio=5 os_prio=0 tid=0x00007ef65eacb000 nid=0x8b29 waiting on condition [0x00007ef6e5e30000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe72516fb8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1057" #667 prio=5 os_prio=0 tid=0x00007ef65eab0000 nid=0x8b28 runnable [0x00007ef6e73d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78260a60> (a sun.nio.ch.Util$3) - locked <0x00007efe78260a78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe782609e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1030" #666 prio=5 os_prio=0 tid=0x00007ef65ea95800 nid=0x8b25 waiting on condition [0x00007ef6f0739000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe914ab4d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1018" #665 prio=5 os_prio=0 tid=0x00007ef65ea7a800 nid=0x8b24 runnable [0x00007ef71f9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf826c10> (a sun.nio.ch.Util$3) - locked <0x00007efdaf826bf8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf821ef0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1025" #664 prio=5 os_prio=0 tid=0x00007ef65ea5f800 nid=0x8b23 runnable [0x00007ef7266fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad338748> (a sun.nio.ch.Util$3) - locked <0x00007efdad338730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad333a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1033" #663 prio=5 os_prio=0 tid=0x00007ef65ea45000 nid=0x8b22 runnable [0x00007ef7267fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9849dd0> (a sun.nio.ch.Util$3) - locked <0x00007efda9849db8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9849d40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1008" #662 prio=5 os_prio=0 tid=0x00007ef65ea2a000 nid=0x8b21 runnable [0x00007ef726e69000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada246b8> (a sun.nio.ch.Util$3) - locked <0x00007efdada246a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada1f900> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1015" #661 prio=5 os_prio=0 tid=0x00007ef65ea0f000 nid=0x8b20 runnable [0x00007ef726f6a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7605a8> (a sun.nio.ch.Util$3) - locked <0x00007efdad760590> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad75b7a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1051" #660 prio=5 os_prio=0 tid=0x00007ef65e9f4000 nid=0x8b1f runnable [0x00007ef72706b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda99ffda8> (a sun.nio.ch.Util$3) - locked <0x00007efda99ffd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99ffd18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1045" #659 prio=5 os_prio=0 tid=0x00007ef65e9d9800 nid=0x8b1e runnable [0x00007ef72abfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a2e8> (a sun.nio.ch.Util$3) - locked <0x00007efda984a2d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1016" #658 prio=5 os_prio=0 tid=0x00007ef65e9be800 nid=0x8b1d runnable [0x00007ef72acfd000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb4039ae8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe72dd5b10> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b80f918> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1060" #657 prio=5 os_prio=0 tid=0x00007ef65e9a4000 nid=0x8b1c waiting on condition [0x00007ef72adfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7244b250> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1061" #656 prio=5 os_prio=0 tid=0x00007ef65e989000 nid=0x8b1b waiting on condition [0x00007ef743fc4000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe723e5348> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1026" #655 prio=5 os_prio=0 tid=0x00007ef65e96e000 nid=0x8b1a runnable [0x00007ef7440c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984a800> (a sun.nio.ch.Util$3) - locked <0x00007efda984a7e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984a770> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1018" #654 prio=5 os_prio=0 tid=0x00007ef65e953800 nid=0x8b19 runnable [0x00007ef7441c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb42252e0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78281640> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91470520> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1011" #653 prio=5 os_prio=0 tid=0x00007ef65e938800 nid=0x8b18 runnable [0x00007ef7b3bbe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf21d9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efdf21d9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf21d97b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1001" #652 prio=5 os_prio=0 tid=0x00007ef65e91d800 nid=0x8b17 runnable [0x00007ef7b3ffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724950> (a sun.nio.ch.Util$3) - locked <0x00007efe7d0d17b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d0d1488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1009" #651 prio=5 os_prio=0 tid=0x00007ef65e903000 nid=0x8b16 runnable [0x00007ef7b81b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf336f18> (a sun.nio.ch.Util$3) - locked <0x00007efdaf336f00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf332160> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1052" #650 prio=5 os_prio=0 tid=0x00007ef65e8e8000 nid=0x8b15 runnable [0x00007ef7b82b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a002c0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a002a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a00230> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1044" #649 prio=5 os_prio=0 tid=0x00007ef65e8cd000 nid=0x8b14 runnable [0x00007ef7b83b8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4953b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab4953a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab495328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1065" #648 prio=5 os_prio=0 tid=0x00007ef65e8b2800 nid=0x8b13 runnable [0x00007ef7b84b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000d40> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1046" #647 prio=5 os_prio=0 tid=0x00007ef65e897800 nid=0x8b12 runnable [0x00007ef7b85ba000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dbcc0> (a sun.nio.ch.Util$3) - locked <0x00007efda98dbca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dbc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1012" #646 prio=5 os_prio=0 tid=0x00007ef65e87c800 nid=0x8b11 runnable [0x00007ef7b86bb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc42648> (a sun.nio.ch.Util$3) - locked <0x00007efdafc42630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc3d928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1011" #645 prio=5 os_prio=0 tid=0x00007ef65e861800 nid=0x8b10 runnable [0x00007ef7b9bbc000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7802b708> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe29013f98> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1027" #644 prio=5 os_prio=0 tid=0x00007ef65e847000 nid=0x8b0e runnable [0x00007ef7bb0bd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab527be0> (a sun.nio.ch.Util$3) - locked <0x00007efdab527bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab527b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1060" #643 prio=5 os_prio=0 tid=0x00007ef65e82c000 nid=0x8b0d runnable [0x00007ef7bb1be000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2475f00> (a sun.nio.ch.Util$3) - locked <0x00007efdf2475f18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2475e88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1070" #642 prio=5 os_prio=0 tid=0x00007ef65e811000 nid=0x8b0c runnable [0x00007ef7bc6bf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6cb4b78> (a sun.nio.ch.Util$3) - locked <0x00007efeb6cb4b90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6cb4b00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1053" #641 prio=5 os_prio=0 tid=0x00007ef65e7f6800 nid=0x8b0b runnable [0x00007ef7d32b7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93760> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93748> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a936d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1011" #640 prio=5 os_prio=0 tid=0x00007ef65e7db800 nid=0x8b0a runnable [0x00007ef7d58f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8374cf68> (a sun.nio.ch.Util$3) - locked <0x00007efe8374cf50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83749b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1034" #639 prio=5 os_prio=0 tid=0x00007ef65e7c1000 nid=0x8b09 runnable [0x00007ef7d7546000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dc608> (a sun.nio.ch.Util$3) - locked <0x00007efda98dc5f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dc578> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1015" #638 prio=5 os_prio=0 tid=0x00007ef65e7a6000 nid=0x8b08 runnable [0x00007ef7d7647000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb454ea18> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78324c58> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c2488d8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1022" #636 prio=5 os_prio=0 tid=0x00007ef65e770000 nid=0x8b06 waiting on condition [0x00007ef7d7849000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528888> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1005" #635 prio=5 os_prio=0 tid=0x00007ef65e755800 nid=0x8b05 runnable [0x00007ef7da201000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d149b0> (a sun.nio.ch.Util$3) - locked <0x00007efd88d14998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d06170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1018" #634 prio=5 os_prio=0 tid=0x00007ef65e73a800 nid=0x8b04 runnable [0x00007ef7e4df8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efeb47b7428> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1066" #633 prio=5 os_prio=0 tid=0x00007ef65e71f800 nid=0x8b03 waiting on condition [0x00007ef7e4ef9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7244b290> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1061" #632 prio=5 os_prio=0 tid=0x00007ef65e704800 nid=0x8b02 runnable [0x00007ef7e4ffa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb7ff00> (a sun.nio.ch.Util$3) - locked <0x00007efdacb7fee8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb7b1e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1024" #631 prio=5 os_prio=0 tid=0x00007ef65e6ea000 nid=0x8b01 runnable [0x00007ef7e50fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984b9d8> (a sun.nio.ch.Util$3) - locked <0x00007efda984b9c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984b948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1064" #630 prio=5 os_prio=0 tid=0x00007ef65e6cf000 nid=0x8b00 runnable [0x00007ef7e51fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a93c78> (a sun.nio.ch.Util$3) - locked <0x00007efda9a93c60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a93be8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1017" #629 prio=5 os_prio=0 tid=0x00007ef65e6b3800 nid=0x8aff runnable [0x00007ef7e52fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2b334e0> (a sun.nio.ch.Util$3) - locked <0x00007efdf2b334c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2b300f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1035" #628 prio=5 os_prio=0 tid=0x00007ef65e698800 nid=0x8afe runnable [0x00007ef7e53fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98dd038> (a sun.nio.ch.Util$3) - locked <0x00007efda98dd020> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98dcfa8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1025" #627 prio=5 os_prio=0 tid=0x00007ef65e67d800 nid=0x8afd waiting on condition [0x00007ef7e54ff000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c671d00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1043" #626 prio=5 os_prio=0 tid=0x00007ef65e662800 nid=0x8afc runnable [0x00007ef7e5600000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6a79e28> (a sun.nio.ch.Util$3) - locked <0x00007efeb6a79e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6a79db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1052" #625 prio=5 os_prio=0 tid=0x00007ef65e647800 nid=0x8afb waiting on condition [0x00007ef7ee3f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b149c00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1035" #623 prio=5 os_prio=0 tid=0x00007ef65e611800 nid=0x8af9 runnable [0x00007ef7ee5fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab001dc0> (a sun.nio.ch.Util$3) - locked <0x00007efdab001da8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab001d30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1012" #622 prio=5 os_prio=0 tid=0x00007ef65e5f7000 nid=0x8af8 runnable [0x00007ef7ee6fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff28453b08> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1001" #620 prio=5 os_prio=0 tid=0x00007ef65e5c1000 nid=0x8af6 runnable [0x00007ef7ee8fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83761ff0> (a sun.nio.ch.Util$3) - locked <0x00007efe83761fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe83761ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1055" #619 prio=5 os_prio=0 tid=0x00007ef65e5a6000 nid=0x8af5 runnable [0x00007ef7ee9fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94190> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94178> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94100> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1006" #618 prio=5 os_prio=0 tid=0x00007ef65e58b800 nid=0x8af4 runnable [0x00007ef7eeaff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6dd2300> (a sun.nio.ch.Util$3) - locked <0x00007efeb6dd2318> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6dd2288> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1028" #617 prio=5 os_prio=0 tid=0x00007ef65e570800 nid=0x8af3 runnable [0x00007ef7eec00000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97b83b0> (a sun.nio.ch.Util$3) - locked <0x00007efda97b8398> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97b8320> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1064" #616 prio=5 os_prio=0 tid=0x00007ef65e555800 nid=0x8af2 runnable [0x00007ef7efb57000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e30> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1051" #615 prio=5 os_prio=0 tid=0x00007ef65e53b000 nid=0x8af1 runnable [0x00007ef7efc58000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab094518> (a sun.nio.ch.Util$3) - locked <0x00007efdab094500> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab094488> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1036" #614 prio=5 os_prio=0 tid=0x00007ef65e520000 nid=0x8af0 runnable [0x00007ef7eff5d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a946a8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94690> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1019" #613 prio=5 os_prio=0 tid=0x00007ef65e505800 nid=0x8aef runnable [0x00007ef7f2480000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7d40d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7d40b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7cf3b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1059" #612 prio=5 os_prio=0 tid=0x00007ef65e4ea800 nid=0x8aee runnable [0x00007ef7f2581000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780000f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1041" #611 prio=5 os_prio=0 tid=0x00007ef65e4cf800 nid=0x8aed runnable [0x00007ef7f2682000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf8d3d20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf8d3d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf8cf000> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1043" #610 prio=5 os_prio=0 tid=0x00007ef65e4b5000 nid=0x8aec runnable [0x00007ef803f52000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0022d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0022c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab002248> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1031" #609 prio=5 os_prio=0 tid=0x00007ef65e49a000 nid=0x8aeb runnable [0x00007ef804053000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a94bc0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a94ba8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a94b30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1032" #608 prio=5 os_prio=0 tid=0x00007ef65e47f000 nid=0x8aea runnable [0x00007ef8056a9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad318b68> (a sun.nio.ch.Util$3) - locked <0x00007efdad318b50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad313db0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1047" #607 prio=5 os_prio=0 tid=0x00007ef65e464000 nid=0x8ae9 runnable [0x00007ef80645a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984cc88> (a sun.nio.ch.Util$3) - locked <0x00007efda984cc70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984cbf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1074" #606 prio=5 os_prio=0 tid=0x00007ef65e449800 nid=0x8ae8 waiting on condition [0x00007ef80670e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe789817f8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1033" #605 prio=5 os_prio=0 tid=0x00007ef65e42e800 nid=0x8ae7 runnable [0x00007ef806975000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95328> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95310> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95298> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1028" #604 prio=5 os_prio=0 tid=0x00007ef65e411800 nid=0x8ae6 runnable [0x00007ef809553000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb34c50> (a sun.nio.ch.Util$3) - locked <0x00007efdadb34c38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb2fe98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1049" #603 prio=5 os_prio=0 tid=0x00007ef65e3f7000 nid=0x8ae5 waiting on condition [0x00007ef816d8f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe724b0f00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1010" #602 prio=5 os_prio=0 tid=0x00007ef65e3d7800 nid=0x8ae4 runnable [0x00007ef817a53000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdac6e90b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1058" #601 prio=5 os_prio=0 tid=0x00007ef65e3bd000 nid=0x8ae3 runnable [0x00007ef818a3e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a7252d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1029" #600 prio=5 os_prio=0 tid=0x00007ef65e3a2000 nid=0x8ae2 waiting on condition [0x00007ef818b3f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000e70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1046" #599 prio=5 os_prio=0 tid=0x00007ef65e387000 nid=0x8ae1 runnable [0x00007ef818c40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4bac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4bab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ba38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1006" #598 prio=5 os_prio=0 tid=0x00007ef65e36c800 nid=0x8ae0 runnable [0x00007ef818d41000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe72daa8b0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c930078> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1069" #597 prio=5 os_prio=0 tid=0x00007ef65e351800 nid=0x8adf waiting on condition [0x00007ef818e42000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c0015b0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1062" #596 prio=5 os_prio=0 tid=0x00007ef65e336800 nid=0x8ade runnable [0x00007ef818f43000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a01fb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a01fa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a01f28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1005" #595 prio=5 os_prio=0 tid=0x00007ef65e31b800 nid=0x8add runnable [0x00007ef819044000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe807aa730> (a sun.nio.ch.Util$3) - locked <0x00007efe807aa718> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe807a7300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1034" #594 prio=5 os_prio=0 tid=0x00007ef65e300800 nid=0x8adc runnable [0x00007ef819145000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a955f0> (a sun.nio.ch.Util$3) - locked <0x00007efda9a955d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a95560> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1060" #593 prio=5 os_prio=0 tid=0x00007ef65e2e5800 nid=0x8adb runnable [0x00007ef819246000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9725fb0> (a sun.nio.ch.Util$3) - locked <0x00007efda9725f98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9725f20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1007" #592 prio=5 os_prio=0 tid=0x00007ef65e2cb000 nid=0x8ada runnable [0x00007ef81b2eb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd304e0> (a sun.nio.ch.Util$3) - locked <0x00007efdafd304c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd2b7c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1018" #591 prio=5 os_prio=0 tid=0x00007ef65e2b0000 nid=0x8ad9 runnable [0x00007ef81b3ec000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5be9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5be990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5b9c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1058" #590 prio=5 os_prio=0 tid=0x00007ef65e295000 nid=0x8ad8 runnable [0x00007ef81b4ed000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c294af0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1009" #589 prio=5 os_prio=0 tid=0x00007ef65e27a800 nid=0x8ad7 runnable [0x00007ef81b5ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe4d4761f8> (a java.lang.Object) - locked <0x00007efe4d4761d8> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1014" #588 prio=5 os_prio=0 tid=0x00007ef65e25f800 nid=0x8ad6 runnable [0x00007ef81b6ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf14ac00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf14abe8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf145ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1009" #587 prio=5 os_prio=0 tid=0x00007ef65e245000 nid=0x8ad5 runnable [0x00007ef81b7f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafcf5388> (a sun.nio.ch.Util$3) - locked <0x00007efdafcf5370> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafcf0668> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1047" #586 prio=5 os_prio=0 tid=0x00007ef65e22a000 nid=0x8ad4 runnable [0x00007ef81b8f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4c4f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4c4e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4c468> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1004" #585 prio=5 os_prio=0 tid=0x00007ef65e20f000 nid=0x8ad3 runnable [0x00007ef81b9f2000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c022990> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1066" #584 prio=5 os_prio=0 tid=0x00007ef65e1f4800 nid=0x8ad2 waiting on condition [0x00007ef81baf3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c528708> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1049" #583 prio=5 os_prio=0 tid=0x00007ef65e1d9800 nid=0x8ad1 runnable [0x00007ef820357000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724a48> (a sun.nio.ch.Util$3) - locked <0x00007efe78724a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe787249d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1060" #582 prio=5 os_prio=0 tid=0x00007ef65e1be800 nid=0x8ad0 runnable [0x00007ef820458000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf223b158> (a sun.nio.ch.Util$3) - locked <0x00007efdf2e08208> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2e07ee0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1045" #581 prio=5 os_prio=0 tid=0x00007ef65e1a3800 nid=0x8acf runnable [0x00007ef823536000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97264c8> (a sun.nio.ch.Util$3) - locked <0x00007efda97264b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9726438> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1035" #580 prio=5 os_prio=0 tid=0x00007ef65e189000 nid=0x8ace runnable [0x00007ef823882000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996ec18> (a sun.nio.ch.Util$3) - locked <0x00007efda996ec00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996eb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1001" #579 prio=5 os_prio=0 tid=0x00007ef65e16e000 nid=0x8acd runnable [0x00007ef823983000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe83277600> (a sun.nio.ch.Util$3) - locked <0x00007efe832775e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe832772f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1044" #578 prio=5 os_prio=0 tid=0x00007ef65e153000 nid=0x8acc runnable [0x00007ef82469b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf38d858> (a sun.nio.ch.Util$3) - locked <0x00007efdaf38d840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf388b38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1070" #577 prio=5 os_prio=0 tid=0x00007ef65e138000 nid=0x8acb runnable [0x00007ef82491e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f279b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f279a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f27928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1027" #576 prio=5 os_prio=0 tid=0x00007ef65e11d800 nid=0x8aca runnable [0x00007ef8255fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e03428> (a sun.nio.ch.Util$3) - locked <0x00007efda9e03410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e03398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1005" #575 prio=5 os_prio=0 tid=0x00007ef65e102800 nid=0x8ac9 runnable [0x00007ef8839f4000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe839bff90> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1052" #574 prio=5 os_prio=0 tid=0x00007ef65e0e8000 nid=0x8ac8 runnable [0x00007ef883dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bb690> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bb678> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bb600> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1025" #573 prio=5 os_prio=0 tid=0x00007ef65e0cd000 nid=0x8ac7 runnable [0x00007ef883efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf43f0e0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf43f0c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf43a3c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1004" #572 prio=5 os_prio=0 tid=0x00007ef65e0b2000 nid=0x8ac6 runnable [0x00007ef883ffe000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91458aa0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1055" #571 prio=5 os_prio=0 tid=0x00007ef65e097800 nid=0x8ac5 runnable [0x00007ef8882ee000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78000200> (a sun.nio.ch.Util$3) - locked <0x00007efe78000218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78000188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1010" #570 prio=5 os_prio=0 tid=0x00007ef65e07c800 nid=0x8ac4 runnable [0x00007ef8883ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe789819e8> (a sun.nio.ch.Util$3) - locked <0x00007efe78981a00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1017" #569 prio=5 os_prio=0 tid=0x00007ef65e062000 nid=0x8ac3 runnable [0x00007ef8884f0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efeb41e9f40> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd506fcbf8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913427c8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1044" #568 prio=5 os_prio=0 tid=0x00007ef65e047000 nid=0x8ac2 runnable [0x00007ef8885f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3aa090> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3aa078> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3a5370> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1045" #567 prio=5 os_prio=0 tid=0x00007ef65e02c000 nid=0x8ac1 runnable [0x00007ef889af3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fa8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1062" #566 prio=5 os_prio=0 tid=0x00007ef65e011000 nid=0x8ac0 runnable [0x00007ef88b9f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984df10> (a sun.nio.ch.Util$3) - locked <0x00007efda984def8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984de80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1035" #565 prio=5 os_prio=0 tid=0x00007ef65dff6800 nid=0x8abf runnable [0x00007ef8900f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a9857a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1053" #564 prio=5 os_prio=0 tid=0x00007ef65dfdb800 nid=0x8abe runnable [0x00007ef890588000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7a081348> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b383058> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1029" #563 prio=5 os_prio=0 tid=0x00007ef65dfc0800 nid=0x8abd waiting on condition [0x00007ef890689000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000358> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1061" #562 prio=5 os_prio=0 tid=0x00007ef65dfa6000 nid=0x8abc runnable [0x00007ef89078a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacbaa0c8> (a sun.nio.ch.Util$3) - locked <0x00007efdacbaa0b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacba53a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1064" #561 prio=5 os_prio=0 tid=0x00007ef65df8b000 nid=0x8abb runnable [0x00007ef89088b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe780002e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1016" #560 prio=5 os_prio=0 tid=0x00007ef65df70000 nid=0x8aba runnable [0x00007ef89c1e2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2c4cbd8> (a sun.nio.ch.Util$3) - locked <0x00007efdf2c4cbc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2c4c8c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1002" #559 prio=5 os_prio=0 tid=0x00007ef65df55000 nid=0x8ab9 runnable [0x00007ef89c2e3000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efeb6a6e508> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1035" #558 prio=5 os_prio=0 tid=0x00007ef65df3a000 nid=0x8ab8 runnable [0x00007ef89c3e4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacb3c9b8> (a sun.nio.ch.Util$3) - locked <0x00007efdacb3c9a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacb37c00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1015" #557 prio=5 os_prio=0 tid=0x00007ef65df1f800 nid=0x8ab7 runnable [0x00007ef89c4e5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad89c358> (a sun.nio.ch.Util$3) - locked <0x00007efdad89c340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad897638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1017" #556 prio=5 os_prio=0 tid=0x00007ef65df04800 nid=0x8ab6 runnable [0x00007ef89c6f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe2c9fc878> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe782fab30> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88d04a68> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1012" #555 prio=5 os_prio=0 tid=0x00007ef65deea000 nid=0x8ab5 runnable [0x00007ef89c7f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae444818> (a sun.nio.ch.Util$3) - locked <0x00007efdae444800> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae43faf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1053" #554 prio=5 os_prio=0 tid=0x00007ef65dece800 nid=0x8ab4 runnable [0x00007ef89c8f5000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b000398> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1039" #553 prio=5 os_prio=0 tid=0x00007ef65deb3800 nid=0x8ab3 waiting on condition [0x00007ef8bb936000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c3fd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1059" #552 prio=5 os_prio=0 tid=0x00007ef65de99000 nid=0x8ab2 runnable [0x00007ef8bd84f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae92ca00> (a sun.nio.ch.Util$3) - locked <0x00007efdae92c9e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae927ce0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1002" #551 prio=5 os_prio=0 tid=0x00007ef65de7e000 nid=0x8ab1 runnable [0x00007ef8c2a2f000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe786132b8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1056" #550 prio=5 os_prio=0 tid=0x00007ef65de63000 nid=0x8ab0 runnable [0x00007ef8c2b30000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9cddc38> (a sun.nio.ch.Util$3) - locked <0x00007efda9cddc20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9cddba8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1054" #549 prio=5 os_prio=0 tid=0x00007ef65de48800 nid=0x8aaf waiting on condition [0x00007ef8c2c31000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78724b28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1039" #548 prio=5 os_prio=0 tid=0x00007ef65de2d800 nid=0x8aac runnable [0x00007ef8c6112000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac5651c0> (a sun.nio.ch.Util$3) - locked <0x00007efdac5651a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac5604a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1042" #547 prio=5 os_prio=0 tid=0x00007ef65de13000 nid=0x8aab runnable [0x00007ef8c6da3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6dd2680> (a sun.nio.ch.Util$3) - locked <0x00007efeb6dd2698> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6dd2608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1066" #546 prio=5 os_prio=0 tid=0x00007ef65ddf8000 nid=0x8aaa runnable [0x00007ef8cdd08000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2476168> (a sun.nio.ch.Util$3) - locked <0x00007efdf2476180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf24760f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1030" #545 prio=5 os_prio=0 tid=0x00007ef65dddd000 nid=0x8aa9 runnable [0x00007ef8cea3f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9bb9ac8> (a sun.nio.ch.Util$3) - locked <0x00007efda9bb9ab0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9bb9a38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1042" #544 prio=5 os_prio=0 tid=0x00007ef65ddc2800 nid=0x8aa8 runnable [0x00007ef8ceb40000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe8013e2d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe913f9898> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1051" #543 prio=5 os_prio=0 tid=0x00007ef65dda7800 nid=0x8aa7 runnable [0x00007ef8cec41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e95c80> (a sun.nio.ch.Util$3) - locked <0x00007efda9e95c68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e95bf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1014" #542 prio=5 os_prio=0 tid=0x00007ef65dd8d000 nid=0x8aa6 runnable [0x00007ef8ced42000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:56) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efed0d24e80> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c4a6400> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1012" #541 prio=5 os_prio=0 tid=0x00007ef65dd72000 nid=0x8aa5 runnable [0x00007ef8d3135000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadb871d0> (a sun.nio.ch.Util$3) - locked <0x00007efdadb871b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadb82418> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1042" #540 prio=5 os_prio=0 tid=0x00007ef65dd57000 nid=0x8aa4 runnable [0x00007ef8dbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda984ee48> (a sun.nio.ch.Util$3) - locked <0x00007efda984ee30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda984edb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1067" #539 prio=5 os_prio=0 tid=0x00007ef65dd3c800 nid=0x8aa3 runnable [0x00007ef8dbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985830> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf223b170> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1012" #538 prio=5 os_prio=0 tid=0x00007ef65dd21800 nid=0x8aa2 runnable [0x00007ef8dc0ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3cad38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3cad20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3c6018> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1020" #537 prio=5 os_prio=0 tid=0x00007ef65dd06800 nid=0x8aa1 runnable [0x00007ef8dc200000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88cac830> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1065" #536 prio=5 os_prio=0 tid=0x00007ef65dceb800 nid=0x8aa0 waiting on condition [0x00007ef8dc301000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0003d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1073" #535 prio=5 os_prio=0 tid=0x00007ef65dcd0800 nid=0x8a9f runnable [0x00007ef8dc402000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf24762f8> (a sun.nio.ch.Util$3) - locked <0x00007efdf31a5b28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf31a5920> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1031" #534 prio=5 os_prio=0 tid=0x00007ef65dcb6000 nid=0x8a9e waiting on condition [0x00007ef8dc503000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672048> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1019" #533 prio=5 os_prio=0 tid=0x00007ef65dc9b000 nid=0x8a9d runnable [0x00007ef8dc604000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf211d868> (a sun.nio.ch.Util$3) - locked <0x00007efdf211d880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf211d7f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1022" #532 prio=5 os_prio=0 tid=0x00007ef65dc80000 nid=0x8a9c runnable [0x00007ef8dc705000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370148> (a sun.nio.ch.Util$3) - locked <0x00007efdab370130> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3700b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1017" #531 prio=5 os_prio=0 tid=0x00007ef65dc65000 nid=0x8a9b runnable [0x00007ef8e44db000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3a7f38> (a sun.nio.ch.Util$3) - locked <0x00007efdae3a7f20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3a7dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1072" #530 prio=5 os_prio=0 tid=0x00007ef65dc4a800 nid=0x8a9a runnable [0x00007ef8e45dc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2000340> (a sun.nio.ch.Util$3) - locked <0x00007efdf2000358> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf20002c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1071" #529 prio=5 os_prio=0 tid=0x00007ef65dc2f800 nid=0x8a99 runnable [0x00007ef8e46dd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4d4b8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4d4a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4d428> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1055" #528 prio=5 os_prio=0 tid=0x00007ef65dc15000 nid=0x8a98 runnable [0x00007ef8e47de000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985940> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985958> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a9858c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1044" #527 prio=5 os_prio=0 tid=0x00007ef65dbfa000 nid=0x8a97 runnable [0x00007ef8e48df000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370660> (a sun.nio.ch.Util$3) - locked <0x00007efdab370648> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3705d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1019" #526 prio=5 os_prio=0 tid=0x00007ef65dbde800 nid=0x8a96 runnable [0x00007ef8e49e0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf03ff78> (a sun.nio.ch.Util$3) - locked <0x00007efdaf03ff60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf03b258> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1038" #525 prio=5 os_prio=0 tid=0x00007ef65dbc4000 nid=0x8a95 runnable [0x00007ef8e4ae1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e96198> (a sun.nio.ch.Util$3) - locked <0x00007efda9e96180> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e96108> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1029" #524 prio=5 os_prio=0 tid=0x00007ef65dba9000 nid=0x8a94 runnable [0x00007ef8e4be2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97ba468> (a sun.nio.ch.Util$3) - locked <0x00007efda97ba450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97ba3d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1008" #523 prio=5 os_prio=0 tid=0x00007ef65db8e000 nid=0x8a93 runnable [0x00007ef8e4ce3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007eff1b9ae268> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1002" #522 prio=5 os_prio=0 tid=0x00007ef65db71000 nid=0x8a92 runnable [0x00007ef8fad2b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efeb50a2b98> (a java.lang.Object) - locked <0x00007efeb50a2b78> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1029" #521 prio=5 os_prio=0 tid=0x00007ef65db56800 nid=0x8a91 runnable [0x00007ef8fdaf1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad814900> (a sun.nio.ch.Util$3) - locked <0x00007efdad8148e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabd26040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1057" #520 prio=5 os_prio=0 tid=0x00007ef65db3b800 nid=0x8a90 waiting on condition [0x00007ef8fea16000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78be20c0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1063" #519 prio=5 os_prio=0 tid=0x00007ef65db20800 nid=0x8a8f runnable [0x00007ef901d5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2358ae8> (a sun.nio.ch.Util$3) - locked <0x00007efdf2358b00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2358a70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1072" #518 prio=5 os_prio=0 tid=0x00007ef65db05800 nid=0x8a8e runnable [0x00007ef910ac6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab370b78> (a sun.nio.ch.Util$3) - locked <0x00007efdab370b60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab370ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1023" #517 prio=5 os_prio=0 tid=0x00007ef65daeb000 nid=0x8a8d runnable [0x00007ef910bc7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad8534d0> (a sun.nio.ch.Util$3) - locked <0x00007efdad8534b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad84e718> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1046" #516 prio=5 os_prio=0 tid=0x00007ef65dad0000 nid=0x8a8c runnable [0x00007ef910cc8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a4c4b60> (a sun.nio.ch.Util$3) - locked <0x00007efe7a4c4b78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a4c4ae8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1059" #515 prio=5 os_prio=0 tid=0x00007ef65dab5000 nid=0x8a8b runnable [0x00007ef910dc9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3eaf38> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3eaf20> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3e6218> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1007" #514 prio=5 os_prio=0 tid=0x00007ef65da9a000 nid=0x8a8a runnable [0x00007ef910eca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafd4d600> (a sun.nio.ch.Util$3) - locked <0x00007efdafd4d5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafd488e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1029" #513 prio=5 os_prio=0 tid=0x00007ef65da7f000 nid=0x8a89 runnable [0x00007ef910fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad589710> (a sun.nio.ch.Util$3) - locked <0x00007efdad5896f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad584a30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1008" #512 prio=5 os_prio=0 tid=0x00007ef65da64800 nid=0x8a88 runnable [0x00007ef9110cc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaecd15c8> (a sun.nio.ch.Util$3) - locked <0x00007efdaecd15b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeccdc30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1053" #511 prio=5 os_prio=0 tid=0x00007ef65da49800 nid=0x8a87 waiting on condition [0x00007ef9111cd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672138> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1015" #510 prio=5 os_prio=0 tid=0x00007ef65da2e800 nid=0x8a86 runnable [0x00007ef9112ce000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf166fe8> (a sun.nio.ch.Util$3) - locked <0x00007efdaf166fd0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1622c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1046" #509 prio=5 os_prio=0 tid=0x00007ef65da13800 nid=0x8a85 runnable [0x00007ef914212000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8bb98760> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1030" #508 prio=5 os_prio=0 tid=0x00007ef65d9f8800 nid=0x8a84 runnable [0x00007ef919306000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4de68> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4de50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ddd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1009" #507 prio=5 os_prio=0 tid=0x00007ef65d9dd800 nid=0x8a83 runnable [0x00007ef919407000] java.lang.Thread.State: RUNNABLE at java.lang.Integer.valueOf(Integer.java:832) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatchSize(ByteBufferLogInputStream.java:87) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:45) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe8032a298> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c638680> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1024" #506 prio=5 os_prio=0 tid=0x00007ef65d9c3000 nid=0x8a82 runnable [0x00007ef919508000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6cb4d38> (a sun.nio.ch.Util$3) - locked <0x00007efeb6cb4d50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6cb4cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1031" #505 prio=5 os_prio=0 tid=0x00007ef65d9a8000 nid=0x8a81 runnable [0x00007ef919609000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac4c64b0> (a sun.nio.ch.Util$3) - locked <0x00007efdac4c6498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac4c16f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1006" #504 prio=5 os_prio=0 tid=0x00007ef65d98d000 nid=0x8a80 runnable [0x00007ef91970a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf26e49e8> (a sun.nio.ch.Util$3) - locked <0x00007efdf26e49d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf26e46d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1068" #503 prio=5 os_prio=0 tid=0x00007ef65d972800 nid=0x8a7f waiting on condition [0x00007ef91980b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b0005a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1041" #502 prio=5 os_prio=0 tid=0x00007ef65d957800 nid=0x8a7e runnable [0x00007ef91990c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6775b8> (a sun.nio.ch.Util$3) - locked <0x00007efdae6775a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae672898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1058" #501 prio=5 os_prio=0 tid=0x00007ef65d93d000 nid=0x8a7d runnable [0x00007ef919a0d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafb21348> (a sun.nio.ch.Util$3) - locked <0x00007efdafb21330> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf08fb28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1063" #500 prio=5 os_prio=0 tid=0x00007ef65d922000 nid=0x8a7c runnable [0x00007ef919b0e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2df588> (a sun.nio.ch.Util$3) - locked <0x00007efdab2df570> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2df4f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1040" #499 prio=5 os_prio=0 tid=0x00007ef65d907000 nid=0x8a7b runnable [0x00007ef922a96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf211da28> (a sun.nio.ch.Util$3) - locked <0x00007efdf211da40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf211d9b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1071" #498 prio=5 os_prio=0 tid=0x00007ef65d8ec000 nid=0x8a7a runnable [0x00007ef922b97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bc158> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bc140> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bc0c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1023" #497 prio=5 os_prio=0 tid=0x00007ef65d8d1800 nid=0x8a79 runnable [0x00007ef92efab000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab003280> (a sun.nio.ch.Util$3) - locked <0x00007efdab003268> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0031f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1054" #496 prio=5 os_prio=0 tid=0x00007ef65d8b6800 nid=0x8a78 runnable [0x00007ef951b9f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6dd2838> (a sun.nio.ch.Util$3) - locked <0x00007efdf3193d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf3193a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1024" #495 prio=5 os_prio=0 tid=0x00007ef65d89b800 nid=0x8a77 runnable [0x00007ef952be4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04698> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04608> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1051" #494 prio=5 os_prio=0 tid=0x00007ef65d881000 nid=0x8a76 runnable [0x00007ef952ce5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad0525b8> (a sun.nio.ch.Util$3) - locked <0x00007efdad0525a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad04d898> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1015" #493 prio=5 os_prio=0 tid=0x00007ef65d866000 nid=0x8a75 runnable [0x00007ef952de6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88c7c848> (a sun.nio.ch.Util$3) - locked <0x00007efd88c7c830> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf36ba878> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1033" #492 prio=5 os_prio=0 tid=0x00007ef65d84b800 nid=0x8a74 waiting on condition [0x00007ef955945000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1008" #491 prio=5 os_prio=0 tid=0x00007ef65d830800 nid=0x8a73 runnable [0x00007ef955a46000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c635498> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1044" #490 prio=5 os_prio=0 tid=0x00007ef65d815800 nid=0x8a72 runnable [0x00007ef955b47000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda996f6e0> (a sun.nio.ch.Util$3) - locked <0x00007efda996f6c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda996f650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1016" #489 prio=5 os_prio=0 tid=0x00007ef65d7fa800 nid=0x8a71 runnable [0x00007ef955c48000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c632650> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1050" #488 prio=5 os_prio=0 tid=0x00007ef65d7df800 nid=0x8a70 waiting on condition [0x00007ef955d49000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efeb6b97360> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1014" #487 prio=5 os_prio=0 tid=0x00007ef65d7c5000 nid=0x8a6f runnable [0x00007ef955ffc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae461af0> (a sun.nio.ch.Util$3) - locked <0x00007efdae461ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae45cdd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1041" #486 prio=5 os_prio=0 tid=0x00007ef65d7aa000 nid=0x8a6e waiting on condition [0x00007ef9560fd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8ba4f960> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1002" #484 prio=5 os_prio=0 tid=0x00007ef65d774800 nid=0x8a6c runnable [0x00007ef99034f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaea1ef18> (a sun.nio.ch.Util$3) - locked <0x00007efdaea1ef00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaea1a1f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1007" #483 prio=5 os_prio=0 tid=0x00007ef65d759800 nid=0x8a6b runnable [0x00007ef990450000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe2ceb8488> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe78c80f18> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efda9cdf438> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1075" #482 prio=5 os_prio=0 tid=0x00007ef65d73f000 nid=0x8a6a runnable [0x00007ef990551000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88e44898> (a sun.nio.ch.Util$3) - locked <0x00007efd88e44880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88e1db30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1016" #481 prio=5 os_prio=0 tid=0x00007ef65d724000 nid=0x8a69 runnable [0x00007ef990652000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbcbcb0> (a sun.nio.ch.Util$3) - locked <0x00007efdadbcbc98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbc6f90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1036" #480 prio=5 os_prio=0 tid=0x00007ef65d709000 nid=0x8a68 runnable [0x00007ef990753000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6dd29f8> (a sun.nio.ch.Util$3) - locked <0x00007efeb6dd2a10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6dd2980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1063" #479 prio=5 os_prio=0 tid=0x00007ef65d6ee000 nid=0x8a67 waiting on condition [0x00007ef990854000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf25935a8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1011" #478 prio=5 os_prio=0 tid=0x00007ef65d6d3800 nid=0x8a66 runnable [0x00007ef990955000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab85f9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab85f990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab85abf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1041" #477 prio=5 os_prio=0 tid=0x00007ef65d6b8800 nid=0x8a65 runnable [0x00007ef991e56000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a04e58> (a sun.nio.ch.Util$3) - locked <0x00007efda9a04e40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a04dc8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1005" #476 prio=5 os_prio=0 tid=0x00007ef65d69d800 nid=0x8a64 runnable [0x00007ef993e54000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88dd4cd8> (a sun.nio.ch.Util$3) - locked <0x00007efd88dd4cc0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88dc1bc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1019" #475 prio=5 os_prio=0 tid=0x00007ef65d682800 nid=0x8a63 runnable [0x00007ef993f55000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab3b53d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab3b53b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab3b0618> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1004" #474 prio=5 os_prio=0 tid=0x00007ef65d668000 nid=0x8a62 runnable [0x00007ef994056000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88d92f08> (a sun.nio.ch.Util$3) - locked <0x00007efd88d92ef0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88d7f598> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1040" #473 prio=5 os_prio=0 tid=0x00007ef65d64d000 nid=0x8a61 runnable [0x00007ef999b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae65c4c0> (a sun.nio.ch.Util$3) - locked <0x00007efdae65c4a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae657708> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1040" #472 prio=5 os_prio=0 tid=0x00007ef65d632800 nid=0x8a60 runnable [0x00007ef99f557000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab4042e0> (a sun.nio.ch.Util$3) - locked <0x00007efdab4042c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404250> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1030" #471 prio=5 os_prio=0 tid=0x00007ef65d617800 nid=0x8a5f runnable [0x00007ef9adb57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5a2958> (a sun.nio.ch.Util$3) - locked <0x00007efdae5a2940> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae59dc38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1072" #470 prio=5 os_prio=0 tid=0x00007ef65d5fc800 nid=0x8a5e runnable [0x00007ef9b57d1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf223b2e8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1040" #469 prio=5 os_prio=0 tid=0x00007ef65d5e1800 nid=0x8a5d runnable [0x00007ef9b5b8e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe78260f28> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1033" #468 prio=5 os_prio=0 tid=0x00007ef65d5c7000 nid=0x8a5c runnable [0x00007ef9b5c8f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bcad8> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bcac0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bca48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1057" #467 prio=5 os_prio=0 tid=0x00007ef65d5ac000 nid=0x8a5b runnable [0x00007ef9b5d90000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d715a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71588> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71510> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1020" #466 prio=5 os_prio=0 tid=0x00007ef65d591000 nid=0x8a5a runnable [0x00007ef9b6555000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe80d02510> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1073" #465 prio=5 os_prio=0 tid=0x00007ef65d576800 nid=0x8a59 runnable [0x00007ef9b6656000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b26b60> (a sun.nio.ch.Util$3) - locked <0x00007efda9b26b48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b26ad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1067" #464 prio=5 os_prio=0 tid=0x00007ef65d55b800 nid=0x8a58 runnable [0x00007ef9b6757000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0a0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1004" #463 prio=5 os_prio=0 tid=0x00007ef65d541000 nid=0x8a57 runnable [0x00007ef9b7b57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6b97480> (a sun.nio.ch.Util$3) - locked <0x00007efeb6b97498> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6b97408> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1018" #462 prio=5 os_prio=0 tid=0x00007ef65d526000 nid=0x8a56 runnable [0x00007ef9c1811000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae3afda8> (a sun.nio.ch.Util$3) - locked <0x00007efdae3afd90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae3ab088> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1042" #461 prio=5 os_prio=0 tid=0x00007ef65d50b000 nid=0x8a55 runnable [0x00007ef9c1912000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf867918> (a sun.nio.ch.Util$3) - locked <0x00007efdaf867900> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf862b60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1037" #460 prio=5 os_prio=0 tid=0x00007ef65d4f0800 nid=0x8a54 runnable [0x00007ef9c1a13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64e4d0> (a sun.nio.ch.Util$3) - locked <0x00007efdab64e4b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64e440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1074" #459 prio=5 os_prio=0 tid=0x00007ef65d4d5800 nid=0x8a53 waiting on condition [0x00007ef9c64eb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b528c70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1014" #458 prio=5 os_prio=0 tid=0x00007ef65d4ba800 nid=0x8a52 runnable [0x00007ef9c6a57000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf6769d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf6769b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf671cb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1012" #457 prio=5 os_prio=0 tid=0x00007ef65d4a0000 nid=0x8a50 runnable [0x00007ef9d8f01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbe93a8> (a sun.nio.ch.Util$3) - locked <0x00007efdafbe9390> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbe46c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1018" #456 prio=5 os_prio=0 tid=0x00007ef65d485000 nid=0x8a4f runnable [0x00007ef9d9002000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efdaf7eb318> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1043" #455 prio=5 os_prio=0 tid=0x00007ef65d46a800 nid=0x8a4e runnable [0x00007ef9d9103000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970270> (a sun.nio.ch.Util$3) - locked <0x00007efda9970258> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99701e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1025" #454 prio=5 os_prio=0 tid=0x00007ef65d44f800 nid=0x8a4d runnable [0x00007ef9dba73000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab095f88> (a sun.nio.ch.Util$3) - locked <0x00007efdab095f70> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab095ef8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1056" #453 prio=5 os_prio=0 tid=0x00007ef65d434800 nid=0x8a4c runnable [0x00007ef9dbb74000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf211de20> (a sun.nio.ch.Util$3) - locked <0x00007efdf211de38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf211dda8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1071" #452 prio=5 os_prio=0 tid=0x00007ef65d419800 nid=0x8a4b runnable [0x00007ef9e369f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe91000f60> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1067" #451 prio=5 os_prio=0 tid=0x00007ef65d3ff000 nid=0x8a4a waiting on condition [0x00007ef9e37a0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1016" #450 prio=5 os_prio=0 tid=0x00007ef65d3e4000 nid=0x8a49 runnable [0x00007ef9f9c4a000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effad2cc140> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe800c2d68> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd88fbc6e0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-7-1069" #449 prio=5 os_prio=0 tid=0x00007ef65d3c9000 nid=0x8a48 runnable [0x00007ef9fba32000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.NativeThread.current(Native Method) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:326) - locked <0x00007efe7c4fe890> (a java.lang.Object) - locked <0x00007efe7c4fe870> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:94) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1032" #448 prio=5 os_prio=0 tid=0x00007ef65d3ae000 nid=0x8a47 runnable [0x00007ef9fcc83000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e052b0> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05298> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05220> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1031" #447 prio=5 os_prio=0 tid=0x00007ef65d393800 nid=0x8a46 runnable [0x00007ef9fdc76000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d71ab8> (a sun.nio.ch.Util$3) - locked <0x00007efda9d71aa0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d71a28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1005" #446 prio=5 os_prio=0 tid=0x00007ef65d378800 nid=0x8a45 runnable [0x00007ef9fdd77000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88cc8858> (a sun.nio.ch.Util$3) - locked <0x00007efd88cc8840> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88cb74d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1004" #445 prio=5 os_prio=0 tid=0x00007ef65d35e000 nid=0x8a44 runnable [0x00007efa092ff000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe80a22d40> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c62c958> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1054" #444 prio=5 os_prio=0 tid=0x00007ef65d343000 nid=0x8a43 runnable [0x00007efa0ac80000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9970788> (a sun.nio.ch.Util$3) - locked <0x00007efda9970770> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99706f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1037" #443 prio=5 os_prio=0 tid=0x00007ef65d328000 nid=0x8a42 runnable [0x00007efa0b362000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0964a0> (a sun.nio.ch.Util$3) - locked <0x00007efdab096488> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096410> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1040" #442 prio=5 os_prio=0 tid=0x00007ef65d30d800 nid=0x8a41 runnable [0x00007efa0b4c1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9f28cb8> (a sun.nio.ch.Util$3) - locked <0x00007efda9f28ca0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9f28c28> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1069" #441 prio=5 os_prio=0 tid=0x00007ef65d2f2800 nid=0x8a40 runnable [0x00007efa1194d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab497a10> (a sun.nio.ch.Util$3) - locked <0x00007efdab4979f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab497980> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1045" #440 prio=5 os_prio=0 tid=0x00007ef65d2d8000 nid=0x8a3f runnable [0x00007efa11a4e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a059e8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a059d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a05958> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1007" #439 prio=5 os_prio=0 tid=0x00007ef65d2bd000 nid=0x8a3e runnable [0x00007efa11c6d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf2fc900> (a sun.nio.ch.Util$3) - locked <0x00007efdaf2fc8e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf2f7be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1004" #438 prio=5 os_prio=0 tid=0x00007ef65d2a2000 nid=0x8a3d runnable [0x00007efa127c6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe80d8eaf0> (a sun.nio.ch.Util$3) - locked <0x00007efe80d8ead8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe80d8e7e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1022" #437 prio=5 os_prio=0 tid=0x00007ef65d287000 nid=0x8a3c runnable [0x00007efa1b78c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97281b8> (a sun.nio.ch.Util$3) - locked <0x00007efda97281a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9728128> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1041" #436 prio=5 os_prio=0 tid=0x00007ef65d26c800 nid=0x8a3b runnable [0x00007efa1be72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4eee0> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4eec8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4ee50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1006" #435 prio=5 os_prio=0 tid=0x00007ef65d251800 nid=0x8a3a runnable [0x00007efa1f8e7000] java.lang.Thread.State: RUNNABLE at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd508f66d0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efd50d3bee0> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) - <0x00007efeb6a740a0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-6-1009" #434 prio=5 os_prio=0 tid=0x00007ef65d237000 nid=0x8a39 runnable [0x00007efa1f9e8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf353a20> (a sun.nio.ch.Util$3) - locked <0x00007efdaf353a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf34ed00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1050" #433 prio=5 os_prio=0 tid=0x00007ef65d21c000 nid=0x8a38 runnable [0x00007efa1fae9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0969b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0969a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab096928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1001" #432 prio=5 os_prio=0 tid=0x00007ef65d201000 nid=0x8a37 runnable [0x00007efa2201a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafc92570> (a sun.nio.ch.Util$3) - locked <0x00007efdafc92558> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafc8d890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1039" #431 prio=5 os_prio=0 tid=0x00007ef65d1e6800 nid=0x8a36 runnable [0x00007efa2211b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e0670> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e0658> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e05e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1043" #430 prio=5 os_prio=0 tid=0x00007ef65d1cb800 nid=0x8a35 runnable [0x00007efa22326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf24764d8> (a sun.nio.ch.Util$3) - locked <0x00007efdf24a2a08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf24a27e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1046" #429 prio=5 os_prio=0 tid=0x00007ef65d1b0800 nid=0x8a34 runnable [0x00007efa22427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f3f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f3e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f368> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1006" #428 prio=5 os_prio=0 tid=0x00007ef65d003800 nid=0x8a33 runnable [0x00007efa226a3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6cb4ef8> (a sun.nio.ch.Util$3) - locked <0x00007efeb6cb4f10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6cb4e80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1035" #427 prio=5 os_prio=0 tid=0x00007ef65d17b000 nid=0x8a32 runnable [0x00007efa28c5f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf27cf1c0> (a sun.nio.ch.Util$3) - locked <0x00007efdf27cf1d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf27cf148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1010" #426 prio=5 os_prio=0 tid=0x00007ef65d160800 nid=0x8a31 runnable [0x00007efa28d60000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efe7a53d820> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe914cce00> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1030" #425 prio=5 os_prio=0 tid=0x00007ef65d145800 nid=0x8a30 waiting on condition [0x00007efa292d2000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe834c0e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1011" #424 prio=5 os_prio=0 tid=0x00007ef65d12a800 nid=0x8a2f runnable [0x00007efa293d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafad2c98> (a sun.nio.ch.Util$3) - locked <0x00007efdafad2c80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafacdfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1052" #423 prio=5 os_prio=0 tid=0x00007ef65d10f800 nid=0x8a2e runnable [0x00007efa34831000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4f910> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4f8f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4f880> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1037" #422 prio=5 os_prio=0 tid=0x00007ef65d0f5000 nid=0x8a2d waiting on condition [0x00007efa34932000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c672308> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1015" #421 prio=5 os_prio=0 tid=0x00007ef65d0da000 nid=0x8a2c runnable [0x00007efa34a33000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbb2568> (a sun.nio.ch.Util$3) - locked <0x00007efdafbb2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf1b34f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1005" #420 prio=5 os_prio=0 tid=0x00007ef65d0bf000 nid=0x8a2b runnable [0x00007efa4029d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe834c0f20> (a sun.nio.ch.Util$3) - locked <0x00007efe834c0f38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe834c0ea8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1047" #419 prio=5 os_prio=0 tid=0x00007ef65d0a4800 nid=0x8a2a runnable [0x00007efa4039e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c4fe28> (a sun.nio.ch.Util$3) - locked <0x00007efda9c4fe10> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c4fd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1020" #418 prio=5 os_prio=0 tid=0x00007ef65d089800 nid=0x8a29 runnable [0x00007efa40911000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf181c70> (a sun.nio.ch.Util$3) - locked <0x00007efdaf181c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf17cf50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1048" #417 prio=5 os_prio=0 tid=0x00007ef65d06f000 nid=0x8a28 runnable [0x00007efa40cca000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bd510> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bd4f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bd480> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1048" #416 prio=5 os_prio=0 tid=0x00007ef65d054000 nid=0x8a27 runnable [0x00007efa4dbf6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab404d50> (a sun.nio.ch.Util$3) - locked <0x00007efdab404d38> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab404cc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1036" #415 prio=5 os_prio=0 tid=0x00007ef65d039000 nid=0x8a26 runnable [0x00007efa4dcf7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24d048> (a sun.nio.ch.Util$3) - locked <0x00007efdab24d030> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24cfb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1027" #414 prio=5 os_prio=0 tid=0x00007ef65d01e800 nid=0x8a25 runnable [0x00007efa4e0a6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c50340> (a sun.nio.ch.Util$3) - locked <0x00007efda9c50328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c502b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1007" #413 prio=5 os_prio=0 tid=0x00007ef65cc59000 nid=0x8a24 runnable [0x00007efa51e86000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.read0(Native Method) at sun.nio.ch.SocketDispatcher.read(SocketDispatcher.java:39) at sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:223) at sun.nio.ch.IOUtil.read(IOUtil.java:197) at sun.nio.ch.SocketChannelImpl.read(SocketChannelImpl.java:380) - locked <0x00007efee23e6488> (a java.lang.Object) at org.apache.kafka.common.network.PlaintextTransportLayer.read(PlaintextTransportLayer.java:104) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:117) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1039" #412 prio=5 os_prio=0 tid=0x00007ef65cfe8800 nid=0x8a23 runnable [0x00007efa5bcb0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7d004600> (a sun.nio.ch.Util$3) - locked <0x00007efe7d004618> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7d004588> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1020" #411 prio=5 os_prio=0 tid=0x00007ef65cfce000 nid=0x8a22 runnable [0x00007efa5bdb1000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8b9bb370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1001" #410 prio=5 os_prio=0 tid=0x00007ef65cfb3000 nid=0x8a21 runnable [0x00007efa5e8a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda987b468> (a sun.nio.ch.Util$3) - locked <0x00007efda987b450> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda98766b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1034" #409 prio=5 os_prio=0 tid=0x00007ef65cf98800 nid=0x8a20 runnable [0x00007efa612f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2476738> (a sun.nio.ch.Util$3) - locked <0x00007efdf2488138> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2487f18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1033" #408 prio=5 os_prio=0 tid=0x00007ef65cf7d800 nid=0x8a1f runnable [0x00007efa642ef000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78724d10> (a sun.nio.ch.Util$3) - locked <0x00007efe78724d28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78724c98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1065" #407 prio=5 os_prio=0 tid=0x00007ef65cf62800 nid=0x8a1e runnable [0x00007efa643f0000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf2358bc8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1021" #406 prio=5 os_prio=0 tid=0x00007ef65cf48000 nid=0x8a1d runnable [0x00007efa644f1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab371900> (a sun.nio.ch.Util$3) - locked <0x00007efdab3718e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab371870> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1006" #405 prio=5 os_prio=0 tid=0x00007ef65cf2d000 nid=0x8a1c runnable [0x00007efa679fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf315e8a8> (a sun.nio.ch.Util$3) - locked <0x00007efdf315e890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf3159b10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1015" #404 prio=5 os_prio=0 tid=0x00007ef65cf12800 nid=0x8a1b runnable [0x00007efa7083b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4059b0> (a sun.nio.ch.Util$3) - locked <0x00007efdae405998> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae400c90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1014" #403 prio=5 os_prio=0 tid=0x00007ef65cef7800 nid=0x8a18 runnable [0x00007efa7093c000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:732) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateProducerState(Log.scala:957) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:849) - locked <0x00007efd5bfd8ba8> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaff48a60> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1057" #402 prio=5 os_prio=0 tid=0x00007ef65cedc800 nid=0x8a17 runnable [0x00007efa70cc6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd88f489b8> (a sun.nio.ch.Util$3) - locked <0x00007efd88f489a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd88f21a68> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1052" #401 prio=5 os_prio=0 tid=0x00007ef65cec2000 nid=0x8a0e runnable [0x00007efa7c1fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a97a98> (a sun.nio.ch.Util$3) - locked <0x00007efda9a97a80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a97a08> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1043" #400 prio=5 os_prio=0 tid=0x00007ef65cea7000 nid=0x8a0d runnable [0x00007efa7c2fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf89e940> (a sun.nio.ch.Util$3) - locked <0x00007efdaf89e928> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf899c20> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1010" #399 prio=5 os_prio=0 tid=0x00007ef65ce8c800 nid=0x8a0c runnable [0x00007efa7c3fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe7a985b00> (a sun.nio.ch.Util$3) - locked <0x00007efe7a985b18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe7a985a88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1010" #398 prio=5 os_prio=0 tid=0x00007ef65ce71800 nid=0x8a0b runnable [0x00007efa7d279000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8ba4fa68> (a sun.nio.ch.Util$3) - locked <0x00007efe8ba4fa80> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8ba4f9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1001" #397 prio=5 os_prio=0 tid=0x00007ef65ce56800 nid=0x8a0a runnable [0x00007efa7d37a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c1a20f0> (a sun.nio.ch.Util$3) - locked <0x00007efe8c1a20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c19d4a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1038" #396 prio=5 os_prio=0 tid=0x00007ef65ce3c000 nid=0x8a09 runnable [0x00007efa7d47b000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda97bbf40> (a sun.nio.ch.Util$3) - locked <0x00007efda97bbf28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97bbeb0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1060" #395 prio=5 os_prio=0 tid=0x00007ef65ce21000 nid=0x8a08 runnable [0x00007efa7d57c000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efeb6a7a008> (a sun.nio.ch.Util$3) - locked <0x00007efeb6b03d40> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6b03b50> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1055" #394 prio=5 os_prio=0 tid=0x00007ef65ce06000 nid=0x8a07 runnable [0x00007efa7d67d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5816d8> (a sun.nio.ch.Util$3) - locked <0x00007efdae5816c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae57c9b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1072" #393 prio=5 os_prio=0 tid=0x00007ef65cdeb800 nid=0x8a06 runnable [0x00007efa7d77e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae68afc8> (a sun.nio.ch.Util$3) - locked <0x00007efdae68afb0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6862a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1071" #392 prio=5 os_prio=0 tid=0x00007ef65cdd0800 nid=0x8a05 runnable [0x00007efa7dd37000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf3c9860> (a sun.nio.ch.Util$3) - locked <0x00007efdaf3c9848> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf3c4b40> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1066" #391 prio=5 os_prio=0 tid=0x00007ef65cdb6000 nid=0x8a04 waiting on condition [0x00007efa7de38000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a2645d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1037" #390 prio=5 os_prio=0 tid=0x00007ef65cd9b000 nid=0x8a02 waiting on condition [0x00007efa7df39000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c7bc108> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1058" #389 prio=5 os_prio=0 tid=0x00007ef65cd80000 nid=0x8a01 runnable [0x00007efa7e03a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae4f7a40> (a sun.nio.ch.Util$3) - locked <0x00007efdae4f7a28> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae4f2c88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1017" #388 prio=5 os_prio=0 tid=0x00007ef65cd65000 nid=0x8a00 runnable [0x00007efa7e431000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf7b8e00> (a sun.nio.ch.Util$3) - locked <0x00007efdaf7b8de8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf7b40e0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1038" #387 prio=5 os_prio=0 tid=0x00007ef65cd4a800 nid=0x89ff runnable [0x00007efa7e532000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b280e0> (a sun.nio.ch.Util$3) - locked <0x00007efda9b280c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28050> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1032" #386 prio=5 os_prio=0 tid=0x00007ef65cd2f800 nid=0x89fe runnable [0x00007efa7e633000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e97b70> (a sun.nio.ch.Util$3) - locked <0x00007efda9e97b58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e97ae0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1052" #385 prio=5 os_prio=0 tid=0x00007ef65cd15000 nid=0x89fd runnable [0x00007efa7e734000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6d1900> (a sun.nio.ch.Util$3) - locked <0x00007efdae6d18e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ccb48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1056" #384 prio=5 os_prio=0 tid=0x00007ef65ccfa000 nid=0x89fc runnable [0x00007efa8c1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b285f8> (a sun.nio.ch.Util$3) - locked <0x00007efda9b285e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28568> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1062" #383 prio=5 os_prio=0 tid=0x00007ef65ccdf000 nid=0x89fb runnable [0x00007efa8d96c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7d0046e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1050" #382 prio=5 os_prio=0 tid=0x00007ef65ccc4800 nid=0x89fa runnable [0x00007efa8dffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e1640> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e1628> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e15b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1037" #381 prio=5 os_prio=0 tid=0x00007ef65cca9800 nid=0x89f9 runnable [0x00007efa9c286000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab498a00> (a sun.nio.ch.Util$3) - locked <0x00007efdab4989e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab498970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1016" #380 prio=5 os_prio=0 tid=0x00007ef65cc8e800 nid=0x89f8 runnable [0x00007efaa4300000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf788540> (a sun.nio.ch.Util$3) - locked <0x00007efdaf788528> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf783860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1075" #379 prio=5 os_prio=0 tid=0x00007ef65cc74000 nid=0x89f7 runnable [0x00007efab4170000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b28b10> (a sun.nio.ch.Util$3) - locked <0x00007efda9b28af8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28a80> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1016" #378 prio=5 os_prio=0 tid=0x00007ef65c197000 nid=0x89f6 runnable [0x00007efab4428000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad707af0> (a sun.nio.ch.Util$3) - locked <0x00007efdad707ad8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad702dd0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1062" #377 prio=5 os_prio=0 tid=0x00007ef65cc3e000 nid=0x89f5 runnable [0x00007efab45d3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd47230> (a sun.nio.ch.Util$3) - locked <0x00007efdacd47218> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdabf71210> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1008" #376 prio=5 os_prio=0 tid=0x00007ef65cc23800 nid=0x89f4 runnable [0x00007efab46d4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed09558> (a sun.nio.ch.Util$3) - locked <0x00007efdaed09540> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed04838> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1005" #375 prio=5 os_prio=0 tid=0x00007ef65cc08800 nid=0x89f3 runnable [0x00007efab6169000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac19f490> (a sun.nio.ch.Util$3) - locked <0x00007efdac19f478> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac19f150> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1008" #374 prio=5 os_prio=0 tid=0x00007ef65cbee000 nid=0x89f2 runnable [0x00007efabc590000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0035c8> (a sun.nio.ch.Util$3) - locked <0x00007efdae0035b0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada5b300> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1009" #373 prio=5 os_prio=0 tid=0x00007ef65cbd3000 nid=0x89f1 runnable [0x00007efabc6f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb7c19f70> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe80ab1e98> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdaeb51428> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1050" #372 prio=5 os_prio=0 tid=0x00007ef65cbb8000 nid=0x89f0 runnable [0x00007efabc950000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdada42358> (a sun.nio.ch.Util$3) - locked <0x00007efdada42340> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdada3d5a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1008" #371 prio=5 os_prio=0 tid=0x00007ef65cb9d000 nid=0x89ef runnable [0x00007efabd5cf000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeced3d0> (a sun.nio.ch.Util$3) - locked <0x00007efdaeced3b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaece86b0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1032" #370 prio=5 os_prio=0 tid=0x00007ef65cb82800 nid=0x89ee runnable [0x00007efabecb7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b29028> (a sun.nio.ch.Util$3) - locked <0x00007efda9b29010> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b28f98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1041" #369 prio=5 os_prio=0 tid=0x00007ef65cb67800 nid=0x89ed runnable [0x00007efac4941000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf323b7c0> (a sun.nio.ch.Util$3) - locked <0x00007efeb6ca0c08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efeb6ca0950> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1053" #368 prio=5 os_prio=0 tid=0x00007ef65cb4c800 nid=0x89ec runnable [0x00007efac4de8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004560> (a sun.nio.ch.Util$3) - locked <0x00007efdab004548> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0044d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1023" #367 prio=5 os_prio=0 tid=0x00007ef65cb31800 nid=0x89e9 runnable [0x00007efac4ee9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8cb98db8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1006" #366 prio=5 os_prio=0 tid=0x00007ef65cb17000 nid=0x89e8 runnable [0x00007efac5196000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effc83426f8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efeb4178250> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1002" #365 prio=5 os_prio=0 tid=0x00007ef65cafc000 nid=0x89e7 runnable [0x00007efac6e4c000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb1f1a230> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd59e99790> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91699840> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1002" #364 prio=5 os_prio=0 tid=0x00007ef65cae1000 nid=0x89e6 runnable [0x00007efac6f4d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf718648> (a sun.nio.ch.Util$3) - locked <0x00007efdaf718630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf713968> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1057" #363 prio=5 os_prio=0 tid=0x00007ef65cac6800 nid=0x89e5 runnable [0x00007efac91d9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdaeaf9a50> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1056" #362 prio=5 os_prio=0 tid=0x00007ef65caab800 nid=0x89e4 runnable [0x00007efad7c96000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab004a78> (a sun.nio.ch.Util$3) - locked <0x00007efdab004a60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0049e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1032" #361 prio=5 os_prio=0 tid=0x00007ef65ca90800 nid=0x89e3 runnable [0x00007efad7d97000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a98ed8> (a sun.nio.ch.Util$3) - locked <0x00007efda9a98ec0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a98e48> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1017" #360 prio=5 os_prio=0 tid=0x00007ef65ca75800 nid=0x89e2 runnable [0x00007efad9225000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae51d880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1050" #359 prio=5 os_prio=0 tid=0x00007ef65ca5a800 nid=0x89e1 runnable [0x00007efad9326000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae0418a8> (a sun.nio.ch.Util$3) - locked <0x00007efdae041890> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae03cb88> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1017" #358 prio=5 os_prio=0 tid=0x00007ef65ca40000 nid=0x89e0 runnable [0x00007efad9427000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaeb37510> (a sun.nio.ch.Util$3) - locked <0x00007efdaeb374f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaeb32830> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1008" #357 prio=5 os_prio=0 tid=0x00007ef65ca25000 nid=0x89df runnable [0x00007efae544a000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe7d36ba80> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdf3a693e8> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-1-1059" #356 prio=5 os_prio=0 tid=0x00007ef65ca0a800 nid=0x89de runnable [0x00007efaf174e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6f20f0> (a sun.nio.ch.Util$3) - locked <0x00007efdae6f20d8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6ed338> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1064" #355 prio=5 os_prio=0 tid=0x00007ef65c9ef800 nid=0x89dd runnable [0x00007efaf1c64000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98590> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98578> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98500> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1004" #354 prio=5 os_prio=0 tid=0x00007ef65c9d4800 nid=0x89dc runnable [0x00007efaf2bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007efe291b1d48> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efeb5b10c08> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdf20f03d0> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1009" #353 prio=5 os_prio=0 tid=0x00007ef65c9ba000 nid=0x89db runnable [0x00007efaf8bc0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafdfc4e8> (a sun.nio.ch.Util$3) - locked <0x00007efdafdfc4d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaecc2860> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1068" #352 prio=5 os_prio=0 tid=0x00007ef65c99f000 nid=0x89da waiting on condition [0x00007efaf8cc1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8c001a88> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1061" #351 prio=5 os_prio=0 tid=0x00007ef65c984800 nid=0x89d9 runnable [0x00007efafa635000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097988> (a sun.nio.ch.Util$3) - locked <0x00007efdab097970> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab0978f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1049" #350 prio=5 os_prio=0 tid=0x00007ef65c969800 nid=0x89d8 runnable [0x00007efafb7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdade7f600> (a sun.nio.ch.Util$3) - locked <0x00007efdade7f5e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdade7a848> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1011" #349 prio=5 os_prio=0 tid=0x00007ef65c94e800 nid=0x89d7 runnable [0x00007efafb8f7000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at scala.collection.TraversableLike$class.last(TraversableLike.scala:432) at scala.collection.AbstractTraversable.last(Traversable.scala:104) at scala.collection.TraversableLike$class.lastOption(TraversableLike.scala:442) at scala.collection.AbstractTraversable.lastOption(Traversable.scala:104) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:181) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe91696b48> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-2-1062" #348 prio=5 os_prio=0 tid=0x00007ef65c934000 nid=0x89d6 runnable [0x00007efafb9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad9f0648> (a sun.nio.ch.Util$3) - locked <0x00007efdad9f0630> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad9eb928> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1061" #347 prio=5 os_prio=0 tid=0x00007ef65c919000 nid=0x89d5 runnable [0x00007efafbaf9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf211df00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1025" #346 prio=5 os_prio=0 tid=0x00007ef65c8fe800 nid=0x89d4 runnable [0x00007efafbbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab372ff0> (a sun.nio.ch.Util$3) - locked <0x00007efdab372fd8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab372f60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1057" #345 prio=5 os_prio=0 tid=0x00007ef65c8e3000 nid=0x89d3 runnable [0x00007efafbcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e98aa8> (a sun.nio.ch.Util$3) - locked <0x00007efda9e98a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e98a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1036" #344 prio=5 os_prio=0 tid=0x00007ef65c8c8800 nid=0x89d2 runnable [0x00007efafbdfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52430> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52418> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c523a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1030" #343 prio=5 os_prio=0 tid=0x00007ef65c8ad800 nid=0x89d1 runnable [0x00007efafbefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab097ea0> (a sun.nio.ch.Util$3) - locked <0x00007efdab097e88> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab097e10> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1048" #342 prio=5 os_prio=0 tid=0x00007ef65c893000 nid=0x89d0 runnable [0x00007efafbffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbefc78> (a sun.nio.ch.Util$3) - locked <0x00007efdadbefc60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadbeaec0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1037" #341 prio=5 os_prio=0 tid=0x00007ef65c878000 nid=0x89cf runnable [0x00007efb242b2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24e9a8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24e990> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24e918> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1021" #340 prio=5 os_prio=0 tid=0x00007ef65c85d800 nid=0x89ce waiting on condition [0x00007efb243b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe784c4190> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1055" #339 prio=5 os_prio=0 tid=0x00007ef65c843800 nid=0x89cd runnable [0x00007efb25f7d000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab64f030> (a sun.nio.ch.Util$3) - locked <0x00007efdab64f018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab64efa0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1028" #338 prio=5 os_prio=0 tid=0x00007ef65c829000 nid=0x89cc runnable [0x00007efb7c5fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9e05d78> (a sun.nio.ch.Util$3) - locked <0x00007efda9e05d60> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9e05ce8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1074" #337 prio=5 os_prio=0 tid=0x00007ef65c80f000 nid=0x89cb runnable [0x00007efb7c6fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae6b1888> (a sun.nio.ch.Util$3) - locked <0x00007efdae6b1870> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae6acad0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1051" #336 prio=5 os_prio=0 tid=0x00007ef65c7f4800 nid=0x89ca runnable [0x00007efb7d1fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9c52948> (a sun.nio.ch.Util$3) - locked <0x00007efda9c52930> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9c528b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1037" #335 prio=5 os_prio=0 tid=0x00007ef65c7da800 nid=0x89c9 runnable [0x00007efb881fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf323b8b8> (a sun.nio.ch.Util$3) - locked <0x00007efdf323b8d0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf323b840> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1019" #334 prio=5 os_prio=0 tid=0x00007ef65c7c0000 nid=0x89c8 runnable [0x00007efb882fd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf9efdb0> (a sun.nio.ch.Util$3) - locked <0x00007efdaf9efd98> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf9eb0d0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1049" #333 prio=5 os_prio=0 tid=0x00007ef65c7a6000 nid=0x89c7 runnable [0x00007efb883fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab0983b8> (a sun.nio.ch.Util$3) - locked <0x00007efdab0983a0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab098328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1063" #332 prio=5 os_prio=0 tid=0x00007ef65c78b800 nid=0x89c6 waiting on condition [0x00007efbb819e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdf223b4f0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1001" #331 prio=5 os_prio=0 tid=0x00007ef65c771000 nid=0x89c5 runnable [0x00007efbb829f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad97e1f8> (a sun.nio.ch.Util$3) - locked <0x00007efdad97e1e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad979518> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1071" #330 prio=5 os_prio=0 tid=0x00007ef65c757000 nid=0x89c4 runnable [0x00007efbb83a0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24eec0> (a sun.nio.ch.Util$3) - locked <0x00007efdab24eea8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24ee30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1026" #329 prio=5 os_prio=0 tid=0x00007ef65c73c800 nid=0x89c3 runnable [0x00007efbb8bd6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729668> (a sun.nio.ch.Util$3) - locked <0x00007efda9729650> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda97295d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1021" #328 prio=5 os_prio=0 tid=0x00007ef65c722800 nid=0x89c2 waiting on condition [0x00007efbb8cd7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df528> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1040" #327 prio=5 os_prio=0 tid=0x00007ef65c708000 nid=0x89c1 runnable [0x00007efbb8f0f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaf883d60> (a sun.nio.ch.Util$3) - locked <0x00007efdaf883d48> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf87f040> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1026" #326 prio=5 os_prio=0 tid=0x00007ef65c6ee000 nid=0x89c0 runnable [0x00007efbb9010000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad873a48> (a sun.nio.ch.Util$3) - locked <0x00007efdad873a30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad86ec90> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1048" #325 prio=5 os_prio=0 tid=0x00007ef65c6d3800 nid=0x89bf runnable [0x00007efbb9111000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78be2538> (a sun.nio.ch.Util$3) - locked <0x00007efe78be2550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78be24c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1010" #324 prio=5 os_prio=0 tid=0x00007ef65c6b9800 nid=0x89be runnable [0x00007efbb95c0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf25936c8> (a sun.nio.ch.Util$3) - locked <0x00007efdf25936e0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2593650> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1007" #323 prio=5 os_prio=0 tid=0x00007ef65c69f000 nid=0x89bd runnable [0x00007efbbb51e000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effb7c4cbe8> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efe72412b40> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8cb21e08> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1056" #322 prio=5 os_prio=0 tid=0x00007ef65c684800 nid=0x89bc runnable [0x00007efbbc01f000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae90c380> (a sun.nio.ch.Util$3) - locked <0x00007efdae90c368> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae9075c8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1034" #321 prio=5 os_prio=0 tid=0x00007ef65c66a800 nid=0x89bb runnable [0x00007efbbc120000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab24f3d8> (a sun.nio.ch.Util$3) - locked <0x00007efdab24f3c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab24f348> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1029" #320 prio=5 os_prio=0 tid=0x00007ef65c650000 nid=0x89b8 runnable [0x00007efbbc221000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9729b80> (a sun.nio.ch.Util$3) - locked <0x00007efda9729b68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9729af0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1023" #319 prio=5 os_prio=0 tid=0x00007ef65c634000 nid=0x89b5 runnable [0x00007efbbc322000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bb428> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bb410> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bb398> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1020" #318 prio=5 os_prio=0 tid=0x00007ef65c619800 nid=0x89b4 runnable [0x00007efbbc423000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad235898> (a sun.nio.ch.Util$3) - locked <0x00007efdad235880> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad230bb8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1016" #317 prio=5 os_prio=0 tid=0x00007ef65c5ff800 nid=0x89b3 runnable [0x00007efbbc6b0000] java.lang.Thread.State: RUNNABLE at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireShared(AbstractQueuedSynchronizer.java:1282) at java.util.concurrent.locks.ReentrantReadWriteLock$ReadLock.lock(ReentrantReadWriteLock.java:727) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:249) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.server.epoch.LeaderEpochFileCache.latestEpoch(LeaderEpochFileCache.scala:78) at kafka.server.epoch.LeaderEpochFileCache.validateAndMaybeWarn(LeaderEpochFileCache.scala:200) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply$mcV$sp(LeaderEpochFileCache.scala:66) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.server.epoch.LeaderEpochFileCache$$anonfun$assign$1.apply(LeaderEpochFileCache.scala:61) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inWriteLock(CoreUtils.scala:259) at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:60) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efe72a648a0> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad50e370> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) - <0x00007efe72a65670> (a java.util.concurrent.locks.ReentrantReadWriteLock$NonfairSync) "ReplicaFetcherThread-6-1019" #316 prio=5 os_prio=0 tid=0x00007ef65c5e5000 nid=0x89b1 runnable [0x00007efbbc7b1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad725318> (a sun.nio.ch.Util$3) - locked <0x00007efdad725300> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7205f8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1010" #314 prio=5 os_prio=0 tid=0x00007ef65c5b0800 nid=0x89af runnable [0x00007efbbca17000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadaffcc0> (a sun.nio.ch.Util$3) - locked <0x00007efdadaffca8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadafda70> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1033" #313 prio=5 os_prio=0 tid=0x00007ef65c596000 nid=0x89ae runnable [0x00007efbbcc13000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9a99e48> (a sun.nio.ch.Util$3) - locked <0x00007efda9a99e30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9a99db8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1007" #312 prio=5 os_prio=0 tid=0x00007ef65c57c000 nid=0x89ad runnable [0x00007efbbce00000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.(HeapByteBuffer.java:57) at java.nio.ByteBuffer.allocate(ByteBuffer.java:335) at org.apache.kafka.common.memory.MemoryPool$1.tryAllocate(MemoryPool.java:30) at org.apache.kafka.common.network.NetworkReceive.readFrom(NetworkReceive.java:112) at org.apache.kafka.common.network.KafkaChannel.receive(KafkaChannel.java:335) at org.apache.kafka.common.network.KafkaChannel.read(KafkaChannel.java:296) at org.apache.kafka.common.network.Selector.attemptRead(Selector.java:560) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:496) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1011" #311 prio=5 os_prio=0 tid=0x00007ef65c561800 nid=0x89ac runnable [0x00007efbbcf01000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe78981e38> (a sun.nio.ch.Util$3) - locked <0x00007efe78981e50> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe78981dc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1068" #310 prio=5 os_prio=0 tid=0x00007ef65c547800 nid=0x89ab runnable [0x00007efbc3efd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2593870> (a sun.nio.ch.Util$3) - locked <0x00007efdf36d22e8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf36d1fd8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1048" #309 prio=5 os_prio=0 tid=0x00007ef65c52d000 nid=0x89aa waiting on condition [0x00007efbc3ffe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdafa7dbc0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1058" #308 prio=5 os_prio=0 tid=0x00007ef65c513000 nid=0x89a9 runnable [0x00007efbc83fe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2e7feb8> (a sun.nio.ch.Util$3) - locked <0x00007efdf348ec68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf348e970> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1039" #307 prio=5 os_prio=0 tid=0x00007ef65c4f8800 nid=0x89a8 runnable [0x00007efbd42b6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b050> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b038> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2afc0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1070" #306 prio=5 os_prio=0 tid=0x00007ef65c4de800 nid=0x89a7 runnable [0x00007efbd5fcb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2b568> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2b550> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b4d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1021" #305 prio=5 os_prio=0 tid=0x00007ef65c4c4000 nid=0x89a6 runnable [0x00007efbd61f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab2e2ca8> (a sun.nio.ch.Util$3) - locked <0x00007efdab2e2c90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab2e2c18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1014" #304 prio=5 os_prio=0 tid=0x00007ef65c4aa000 nid=0x89a5 runnable [0x00007efbd62f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafddad80> (a sun.nio.ch.Util$3) - locked <0x00007efdafddad68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafdd60a0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1021" #303 prio=5 os_prio=0 tid=0x00007ef65c48f800 nid=0x89a4 waiting on condition [0x00007efbd63f8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4dd0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1042" #302 prio=5 os_prio=0 tid=0x00007ef65c475000 nid=0x89a3 runnable [0x00007efbd64f9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9b2ba80> (a sun.nio.ch.Util$3) - locked <0x00007efda9b2ba68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9b2b9f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1019" #301 prio=5 os_prio=0 tid=0x00007ef65c45a800 nid=0x89a2 runnable [0x00007efbd65fa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdae5da2e0> (a sun.nio.ch.Util$3) - locked <0x00007efdae5da2c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdae5d55c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1018" #300 prio=5 os_prio=0 tid=0x00007ef65c440800 nid=0x89a1 runnable [0x00007efbd68fb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf323bac8> (a sun.nio.ch.Util$3) - locked <0x00007efdf323bae0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaf5c5cf0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1045" #299 prio=5 os_prio=0 tid=0x00007ef65c426000 nid=0x89a0 runnable [0x00007efbd69fc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda98519a0> (a sun.nio.ch.Util$3) - locked <0x00007efda9851988> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9851910> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1023" #298 prio=5 os_prio=0 tid=0x00007ef65c40c000 nid=0x899f waiting on condition [0x00007efbd6afd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe8b3df568> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1026" #297 prio=5 os_prio=0 tid=0x00007ef65c3f1800 nid=0x899c runnable [0x00007efbd6bfe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab5bdf30> (a sun.nio.ch.Util$3) - locked <0x00007efdab5bdf18> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab5bdea0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1006" #296 prio=5 os_prio=0 tid=0x00007ef65c3d7800 nid=0x899b runnable [0x00007efbd7194000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdac7f3120> (a sun.nio.ch.Util$3) - locked <0x00007efdac7f3108> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdac7ee440> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1051" #295 prio=5 os_prio=0 tid=0x00007ef65c3bd000 nid=0x899a runnable [0x00007efbd7295000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9d73748> (a sun.nio.ch.Util$3) - locked <0x00007efda9d73730> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda9d736b8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1014" #294 prio=5 os_prio=0 tid=0x00007ef65c3a3000 nid=0x8999 runnable [0x00007efbe41f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdafbccf20> (a sun.nio.ch.Util$3) - locked <0x00007efdafbccf08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdafbc8200> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1002" #293 prio=5 os_prio=0 tid=0x00007ef65c388800 nid=0x8998 runnable [0x00007efbe42f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad7af7d8> (a sun.nio.ch.Util$3) - locked <0x00007efdad7af7c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad7aaaf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1002" #292 prio=5 os_prio=0 tid=0x00007ef65c36e800 nid=0x8997 runnable [0x00007efbe447a000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Checksums.update(Checksums.java:42) at org.apache.kafka.common.utils.Crc32C.compute(Crc32C.java:72) at org.apache.kafka.common.record.DefaultRecordBatch.computeChecksum(DefaultRecordBatch.java:374) at org.apache.kafka.common.record.DefaultRecordBatch.isValid(DefaultRecordBatch.java:370) at org.apache.kafka.common.record.DefaultRecordBatch.ensureValid(DefaultRecordBatch.java:148) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1041) at kafka.log.Log$$anonfun$kafka$log$Log$$analyzeAndValidateRecords$1.apply(Log.scala:1004) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe78c20e18> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-0-1062" #291 prio=5 os_prio=0 tid=0x00007ef65c354000 nid=0x8996 waiting on condition [0x00007efbe580f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a4c4df8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1020" #290 prio=5 os_prio=0 tid=0x00007ef65c33a000 nid=0x8995 runnable [0x00007efbe5910000] java.lang.Thread.State: RUNNABLE at java.nio.Bits.copyFromArray(Bits.java:808) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:373) at java.nio.DirectByteBuffer.put(DirectByteBuffer.java:342) at sun.nio.ch.IOUtil.write(IOUtil.java:60) at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211) - locked <0x00007effd31f29a0> (a java.lang.Object) at org.apache.kafka.common.record.MemoryRecords.writeFullyTo(MemoryRecords.java:95) at org.apache.kafka.common.record.FileRecords.append(FileRecords.java:151) at kafka.log.LogSegment.append(LogSegment.scala:138) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:866) - locked <0x00007efd5b0bb068> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdad1da880> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-5-1017" #289 prio=5 os_prio=0 tid=0x00007ef65c31f800 nid=0x8994 runnable [0x00007efbe5a11000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdadbaa5d8> (a sun.nio.ch.Util$3) - locked <0x00007efdadbaa5c0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdadba5820> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1034" #288 prio=5 os_prio=0 tid=0x00007ef65c305800 nid=0x8993 runnable [0x00007efbe5dd0000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2e7ffb0> (a sun.nio.ch.Util$3) - locked <0x00007efdf2e7ffc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2e7ff38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1070" #287 prio=5 os_prio=0 tid=0x00007ef65c2eb000 nid=0x8992 runnable [0x00007efbe60e7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2d62370> (a sun.nio.ch.Util$3) - locked <0x00007efdf2dcfa30> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2dcf810> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1036" #286 prio=5 os_prio=0 tid=0x00007ef65c2d1000 nid=0x8991 runnable [0x00007efbe6f91000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf323bbd8> (a sun.nio.ch.Util$3) - locked <0x00007efdf323bbf0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf323bb60> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1011" #285 prio=5 os_prio=0 tid=0x00007ef65c2b6000 nid=0x8990 runnable [0x00007efbe7175000] java.lang.Thread.State: RUNNABLE at kafka.server.epoch.LeaderEpochFileCache.assign(LeaderEpochFileCache.scala:61) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:838) at kafka.log.Log$$anonfun$append$2$$anonfun$apply$14.apply(Log.scala:836) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:836) - locked <0x00007efd58a54608> (a java.lang.Object) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efdae889128> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-4-1050" #284 prio=5 os_prio=0 tid=0x00007ef65c29c000 nid=0x898f waiting on condition [0x00007efbe7276000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe7a264600> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-1-1012" #283 prio=5 os_prio=0 tid=0x00007ef65c281800 nid=0x898e runnable [0x00007efbfc343000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdad741410> (a sun.nio.ch.Util$3) - locked <0x00007efdad7413f8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdad73c6f0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-6-1001" #282 prio=5 os_prio=0 tid=0x00007ef65c267800 nid=0x898d runnable [0x00007efbfc444000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efe8c24f698> (a sun.nio.ch.Util$3) - locked <0x00007efe8c24f680> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efe8c24c1d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-0-1005" #281 prio=5 os_prio=0 tid=0x00007ef65c24d000 nid=0x898c runnable [0x00007efbfc545000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdacd650e0> (a sun.nio.ch.Util$3) - locked <0x00007efdacd650c8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdacd60328> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1012" #280 prio=5 os_prio=0 tid=0x00007ef65c232800 nid=0x8981 runnable [0x00007efbfc646000] java.lang.Thread.State: RUNNABLE at java.nio.HeapByteBuffer.slice(HeapByteBuffer.java:98) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:51) at org.apache.kafka.common.record.ByteBufferLogInputStream.nextBatch(ByteBufferLogInputStream.java:33) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:35) at org.apache.kafka.common.record.RecordBatchIterator.makeNext(RecordBatchIterator.java:24) at org.apache.kafka.common.utils.AbstractIterator.maybeComputeNext(AbstractIterator.java:79) at org.apache.kafka.common.utils.AbstractIterator.hasNext(AbstractIterator.java:45) at scala.collection.convert.Wrappers$JIteratorWrapper.hasNext(Wrappers.scala:42) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.log.Log.kafka$log$Log$$analyzeAndValidateRecords(Log.scala:1004) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:753) at kafka.log.Log$$anonfun$append$2.apply(Log.scala:752) at kafka.log.Log.maybeHandleIOException(Log.scala:1839) at kafka.log.Log.append(Log.scala:752) at kafka.log.Log.appendAsFollower(Log.scala:733) at kafka.cluster.Partition$$anonfun$doAppendRecordsToFollowerOrFutureReplica$1.apply(Partition.scala:589) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.utils.CoreUtils$.inReadLock(CoreUtils.scala:257) at kafka.cluster.Partition.doAppendRecordsToFollowerOrFutureReplica(Partition.scala:576) at kafka.cluster.Partition.appendRecordsToFollowerOrFutureReplica(Partition.scala:596) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:129) at kafka.server.ReplicaFetcherThread.processPartitionData(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:186) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1$$anonfun$apply$2.apply(AbstractFetcherThread.scala:172) at scala.Option.foreach(Option.scala:257) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:172) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2$$anonfun$apply$mcV$sp$1.apply(AbstractFetcherThread.scala:169) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply$mcV$sp(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.server.AbstractFetcherThread$$anonfun$processFetchRequest$2.apply(AbstractFetcherThread.scala:169) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:167) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - <0x00007efe8c1c1118> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) "ReplicaFetcherThread-3-1039" #279 prio=5 os_prio=0 tid=0x00007ef65c218800 nid=0x897e waiting on condition [0x00007efbfc747000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe80263650> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1007" #278 prio=5 os_prio=0 tid=0x00007ef65c1fe000 nid=0x897d runnable [0x00007efbfc848000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdaed405b0> (a sun.nio.ch.Util$3) - locked <0x00007efdaed40598> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdaed3b890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1065" #277 prio=5 os_prio=0 tid=0x00007ef65c1e4800 nid=0x897c runnable [0x00007efbfc949000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdab1bc708> (a sun.nio.ch.Util$3) - locked <0x00007efdab1bc6f0> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdab1bc678> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-4-1054" #276 prio=5 os_prio=0 tid=0x00007ef65c1ca800 nid=0x8923 runnable [0x00007efbfd64a000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda9972450> (a sun.nio.ch.Util$3) - locked <0x00007efda9972438> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda99723c0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-3-1055" #275 prio=5 os_prio=0 tid=0x00007ef65c1b1000 nid=0x8922 runnable [0x00007efbfda72000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efda972ad90> (a sun.nio.ch.Util$3) - locked <0x00007efda972ad78> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efda972ad00> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-2-1047" #274 prio=5 os_prio=0 tid=0x00007ef65c162000 nid=0x8921 runnable [0x00007efbfe082000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efdf2b273f0> (a sun.nio.ch.Util$3) - locked <0x00007efdf2b27408> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efdf2b27378> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at org.apache.kafka.clients.NetworkClientUtils.sendAndReceive(NetworkClientUtils.java:93) at kafka.server.ReplicaFetcherBlockingSend.sendRequest(ReplicaFetcherBlockingSend.scala:96) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:240) at kafka.server.ReplicaFetcherThread.fetch(ReplicaFetcherThread.scala:43) at kafka.server.AbstractFetcherThread.processFetchRequest(AbstractFetcherThread.scala:149) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:114) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-7-1025" #273 prio=5 os_prio=0 tid=0x00007ef65c163000 nid=0x8920 waiting on condition [0x00007efbfed2b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad4d2688> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ReplicaFetcherThread-5-1047" #272 prio=5 os_prio=0 tid=0x00007ef65c15f800 nid=0x891f waiting on condition [0x00007ef53aebd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efdad1d6ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2163) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:108) at kafka.server.AbstractFetcherThread$$anonfun$2.apply(AbstractFetcherThread.scala:104) at kafka.utils.CoreUtils$.inLock(CoreUtils.scala:251) at kafka.server.AbstractFetcherThread.doWork(AbstractFetcherThread.scala:104) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-23" #269 prio=5 os_prio=0 tid=0x00007f0649b12800 nid=0x8865 runnable [0x00007efbff3f2000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674340> (a sun.nio.ch.Util$3) - locked <0x00007efd76674328> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674358> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-22" #268 prio=5 os_prio=0 tid=0x00007f0649b11800 nid=0x8864 runnable [0x00007efbff4f3000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674790> (a sun.nio.ch.Util$3) - locked <0x00007efd76674778> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766747a8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-21" #267 prio=5 os_prio=0 tid=0x00007f0649b10000 nid=0x8863 runnable [0x00007efbff5f4000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76674be0> (a sun.nio.ch.Util$3) - locked <0x00007efd76674bc8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76674bf8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-20" #266 prio=5 os_prio=0 tid=0x00007f0649b0f000 nid=0x8862 runnable [0x00007efbff6f5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675030> (a sun.nio.ch.Util$3) - locked <0x00007efd76675018> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675048> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-19" #265 prio=5 os_prio=0 tid=0x00007f0649b0d800 nid=0x8861 runnable [0x00007efbff7f6000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675480> (a sun.nio.ch.Util$3) - locked <0x00007efd76675468> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675498> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-18" #264 prio=5 os_prio=0 tid=0x00007f0649b0c800 nid=0x8860 runnable [0x00007efbff8f7000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766758d0> (a sun.nio.ch.Util$3) - locked <0x00007efd766758b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766758e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-17" #263 prio=5 os_prio=0 tid=0x00007f0649b0b800 nid=0x885f runnable [0x00007efbff9f8000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76675d20> (a sun.nio.ch.Util$3) - locked <0x00007efd76675d08> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76675d38> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-16" #262 prio=5 os_prio=0 tid=0x00007f0649b31800 nid=0x885e runnable [0x00007efbffaf9000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fc530> (a sun.nio.ch.Util$3) - locked <0x00007efd766fc518> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fc548> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-15" #261 prio=5 os_prio=0 tid=0x00007f0649b30000 nid=0x885d runnable [0x00007efbffbfa000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76676170> (a sun.nio.ch.Util$3) - locked <0x00007efd76676158> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76676188> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-14" #260 prio=5 os_prio=0 tid=0x00007f0649b2f000 nid=0x885c runnable [0x00007efbffcfb000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766765c0> (a sun.nio.ch.Util$3) - locked <0x00007efd766765a8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766765d8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-13" #259 prio=5 os_prio=0 tid=0x00007f0649b2e000 nid=0x885b waiting for monitor entry [0x00007efbffdfc000] java.lang.Thread.State: BLOCKED (on object monitor) at java.security.Provider.getService(Provider.java:1035) - waiting to lock <0x00007efd503a2538> (a com.sun.crypto.provider.SunJCE) at sun.security.jca.ProviderList$ServiceList.tryGet(ProviderList.java:444) at sun.security.jca.ProviderList$ServiceList.access$200(ProviderList.java:376) at sun.security.jca.ProviderList$ServiceList$1.hasNext(ProviderList.java:486) at javax.crypto.Cipher.getInstance(Cipher.java:512) at sun.security.krb5.internal.crypto.dk.AesDkCrypto.getCipher(AesDkCrypto.java:148) at sun.security.krb5.internal.crypto.dk.DkCrypto.dr(DkCrypto.java:484) at sun.security.krb5.internal.crypto.dk.DkCrypto.dk(DkCrypto.java:447) at sun.security.krb5.internal.crypto.dk.AesDkCrypto.calculateChecksum(AesDkCrypto.java:198) at sun.security.krb5.internal.crypto.Aes256.calculateChecksum(Aes256.java:56) at sun.security.jgss.krb5.CipherHelper.calculateChecksum(CipherHelper.java:357) at sun.security.jgss.krb5.MessageToken_v2.getChecksum(MessageToken_v2.java:430) at sun.security.jgss.krb5.MessageToken_v2.genSignAndSeqNumber(MessageToken_v2.java:331) at sun.security.jgss.krb5.WrapToken_v2.(WrapToken_v2.java:173) at sun.security.jgss.krb5.Krb5Context.wrap(Krb5Context.java:926) at sun.security.jgss.GSSContextImpl.wrap(GSSContextImpl.java:385) at com.sun.security.sasl.gsskerb.GssKrb5Server.doHandshake1(GssKrb5Server.java:235) at com.sun.security.sasl.gsskerb.GssKrb5Server.evaluateResponse(GssKrb5Server.java:193) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.handleSaslToken(SaslServerAuthenticator.java:377) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.authenticate(SaslServerAuthenticator.java:261) at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:127) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:487) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-12" #258 prio=5 os_prio=0 tid=0x00007f0649b2d000 nid=0x885a runnable [0x00007efbffefd000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685d930> (a sun.nio.ch.Util$3) - locked <0x00007efd7685d918> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685d948> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-11" #257 prio=5 os_prio=0 tid=0x00007f0649b2b800 nid=0x8859 runnable [0x00007efbffffe000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685dd80> (a sun.nio.ch.Util$3) - locked <0x00007efd7685dd68> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685dd98> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-10" #256 prio=5 os_prio=0 tid=0x00007f0649b2b000 nid=0x8858 runnable [0x00007efc04513000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e1d0> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e1b8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e1e8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-9" #255 prio=5 os_prio=0 tid=0x00007f064a5bf800 nid=0x8857 runnable [0x00007efc0471a000] java.lang.Thread.State: RUNNABLE at javax.crypto.spec.IvParameterSpec.getIV(IvParameterSpec.java:97) at com.sun.crypto.provider.CipherCore.init(CipherCore.java:523) at com.sun.crypto.provider.AESCipher.engineInit(AESCipher.java:346) at javax.crypto.Cipher.implInit(Cipher.java:805) at javax.crypto.Cipher.chooseProvider(Cipher.java:863) - locked <0x00007f04ec002458> (a java.lang.Object) at javax.crypto.Cipher.init(Cipher.java:1395) at javax.crypto.Cipher.init(Cipher.java:1326) at sun.security.krb5.internal.crypto.dk.AesDkCrypto.decryptCTS(AesDkCrypto.java:413) at sun.security.krb5.internal.crypto.dk.AesDkCrypto.decrypt(AesDkCrypto.java:272) at sun.security.krb5.internal.crypto.Aes256.decrypt(Aes256.java:76) at sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:100) at sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:94) at sun.security.krb5.EncryptedData.decrypt(EncryptedData.java:175) at sun.security.krb5.KrbApReq.authenticate(KrbApReq.java:288) at sun.security.krb5.KrbApReq.(KrbApReq.java:149) at sun.security.jgss.krb5.InitSecContextToken.(InitSecContextToken.java:108) at sun.security.jgss.krb5.Krb5Context.acceptSecContext(Krb5Context.java:829) at sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:342) at sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:285) at com.sun.security.sasl.gsskerb.GssKrb5Server.evaluateResponse(GssKrb5Server.java:167) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.handleSaslToken(SaslServerAuthenticator.java:377) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.authenticate(SaslServerAuthenticator.java:261) at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:127) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:487) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-8" #254 prio=5 os_prio=0 tid=0x00007f064a5be800 nid=0x8856 runnable [0x00007efc054d1000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7685e620> (a sun.nio.ch.Util$3) - locked <0x00007efd7685e608> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7685e638> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-7" #253 prio=5 os_prio=0 tid=0x00007f064a5bd800 nid=0x8855 runnable [0x00007efc0593b000] java.lang.Thread.State: RUNNABLE at org.apache.kafka.common.utils.Utils.utf8(Utils.java:136) at org.apache.kafka.common.utils.Utils.utf8(Utils.java:111) at org.apache.kafka.common.protocol.types.Type$8.read(Type.java:391) at org.apache.kafka.common.protocol.types.Type$8.read(Type.java:364) at org.apache.kafka.common.protocol.types.Schema.read(Schema.java:74) at org.apache.kafka.common.requests.RequestHeader.parse(RequestHeader.java:121) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.handleSaslToken(SaslServerAuthenticator.java:358) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.authenticate(SaslServerAuthenticator.java:261) at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:127) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:487) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6" #252 prio=5 os_prio=0 tid=0x00007f064a5bd000 nid=0x8854 waiting for monitor entry [0x00007efc05a3c000] java.lang.Thread.State: BLOCKED (on object monitor) at java.security.Provider.getService(Provider.java:1035) - locked <0x00007efd503a2538> (a com.sun.crypto.provider.SunJCE) at sun.security.jca.ProviderList$ServiceList.tryGet(ProviderList.java:444) at sun.security.jca.ProviderList$ServiceList.access$200(ProviderList.java:376) at sun.security.jca.ProviderList$ServiceList$1.hasNext(ProviderList.java:486) at javax.crypto.Cipher.getInstance(Cipher.java:512) at sun.security.krb5.internal.crypto.dk.AesDkCrypto.getCipher(AesDkCrypto.java:148) at sun.security.krb5.internal.crypto.dk.DkCrypto.dr(DkCrypto.java:484) at sun.security.krb5.internal.crypto.dk.DkCrypto.dk(DkCrypto.java:447) at sun.security.krb5.internal.crypto.dk.AesDkCrypto.decryptCTS(AesDkCrypto.java:394) at sun.security.krb5.internal.crypto.dk.AesDkCrypto.decrypt(AesDkCrypto.java:272) at sun.security.krb5.internal.crypto.Aes256.decrypt(Aes256.java:76) at sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:100) at sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:94) at sun.security.krb5.EncryptedData.decrypt(EncryptedData.java:175) at sun.security.krb5.KrbApReq.authenticate(KrbApReq.java:288) at sun.security.krb5.KrbApReq.(KrbApReq.java:149) at sun.security.jgss.krb5.InitSecContextToken.(InitSecContextToken.java:108) at sun.security.jgss.krb5.Krb5Context.acceptSecContext(Krb5Context.java:829) at sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:342) at sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:285) at com.sun.security.sasl.gsskerb.GssKrb5Server.evaluateResponse(GssKrb5Server.java:167) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.handleSaslToken(SaslServerAuthenticator.java:377) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.authenticate(SaslServerAuthenticator.java:261) at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:127) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:487) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-5" #251 prio=5 os_prio=0 tid=0x00007f064a5bc000 nid=0x8853 runnable [0x00007efc05b3d000] java.lang.Thread.State: RUNNABLE at java.lang.System.identityHashCode(Native Method) at java.util.IdentityHashMap.hash(IdentityHashMap.java:295) at java.util.IdentityHashMap.get(IdentityHashMap.java:329) at javax.crypto.JceSecurity.getVerificationResult(JceSecurity.java:176) - locked <0x00007efd5035b120> (a java.lang.Class for javax.crypto.JceSecurity) at javax.crypto.JceSecurity.canUseProvider(JceSecurity.java:204) at javax.crypto.Cipher.getInstance(Cipher.java:514) at sun.security.krb5.internal.crypto.dk.AesDkCrypto.decryptCTS(AesDkCrypto.java:410) at sun.security.krb5.internal.crypto.dk.AesDkCrypto.decrypt(AesDkCrypto.java:272) at sun.security.krb5.internal.crypto.Aes256.decrypt(Aes256.java:76) at sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:100) at sun.security.krb5.internal.crypto.Aes256CtsHmacSha1EType.decrypt(Aes256CtsHmacSha1EType.java:94) at sun.security.krb5.EncryptedData.decrypt(EncryptedData.java:175) at sun.security.krb5.KrbApReq.authenticate(KrbApReq.java:288) at sun.security.krb5.KrbApReq.(KrbApReq.java:149) at sun.security.jgss.krb5.InitSecContextToken.(InitSecContextToken.java:108) at sun.security.jgss.krb5.Krb5Context.acceptSecContext(Krb5Context.java:829) at sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:342) at sun.security.jgss.GSSContextImpl.acceptSecContext(GSSContextImpl.java:285) at com.sun.security.sasl.gsskerb.GssKrb5Server.evaluateResponse(GssKrb5Server.java:167) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.handleSaslToken(SaslServerAuthenticator.java:377) at org.apache.kafka.common.security.authenticator.SaslServerAuthenticator.authenticate(SaslServerAuthenticator.java:261) at org.apache.kafka.common.network.KafkaChannel.prepare(KafkaChannel.java:127) at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:487) at org.apache.kafka.common.network.Selector.poll(Selector.java:425) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-4" #250 prio=5 os_prio=0 tid=0x00007f064a5bb800 nid=0x8852 runnable [0x00007efc05c3e000] java.lang.Thread.State: RUNNABLE at java.lang.Object.hashCode(Native Method) at java.util.HashMap.hash(HashMap.java:339) at java.util.HashMap.put(HashMap.java:612) at java.util.HashSet.add(HashSet.java:220) at java.nio.channels.spi.AbstractSelector.cancel(AbstractSelector.java:92) - locked <0x00007efd83a81ef8> (a java.util.HashSet) at java.nio.channels.spi.AbstractSelectionKey.cancel(AbstractSelectionKey.java:73) - locked <0x00007f04ecab8e20> (a sun.nio.ch.SelectionKeyImpl) at org.apache.kafka.common.network.PlaintextTransportLayer.disconnect(PlaintextTransportLayer.java:58) at org.apache.kafka.common.network.KafkaChannel.disconnect(KafkaChannel.java:140) at org.apache.kafka.common.network.Selector.close(Selector.java:721) at org.apache.kafka.common.network.Selector.close(Selector.java:701) at kafka.network.Processor$$anonfun$kafka$network$Processor$$close$1.apply(SocketServer.scala:771) at kafka.network.Processor$$anonfun$kafka$network$Processor$$close$1.apply(SocketServer.scala:766) at scala.Option.foreach(Option.scala:257) at kafka.network.Processor.kafka$network$Processor$$close(SocketServer.scala:766) at kafka.network.Processor.processNewResponses(SocketServer.scala:641) at kafka.network.Processor.run(SocketServer.scala:582) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-3" #249 prio=5 os_prio=0 tid=0x00007f064a5ba800 nid=0x8851 waiting on condition [0x00007efc05d3f000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.put(ArrayBlockingQueue.java:350) at kafka.network.RequestChannel.sendRequest(RequestChannel.scala:310) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:698) at kafka.network.Processor$$anonfun$processCompletedReceives$1.apply(SocketServer.scala:688) at scala.collection.Iterator$class.foreach(Iterator.scala:891) at scala.collection.AbstractIterator.foreach(Iterator.scala:1334) at scala.collection.IterableLike$class.foreach(IterableLike.scala:72) at scala.collection.AbstractIterable.foreach(Iterable.scala:54) at kafka.network.Processor.processCompletedReceives(SocketServer.scala:688) at kafka.network.Processor.run(SocketServer.scala:584) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-2" #248 prio=5 os_prio=0 tid=0x00007f064a5ba000 nid=0x8850 runnable [0x00007efc05e40000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd7678de18> (a sun.nio.ch.Util$3) - locked <0x00007efd7678de00> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd7678de30> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-1" #247 prio=5 os_prio=0 tid=0x00007f064a5b9000 nid=0x884f runnable [0x00007efc05f41000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766fcdd0> (a sun.nio.ch.Util$3) - locked <0x00007efd766fcdb8> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766fcde8> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-network-thread-1013-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-0" #246 prio=5 os_prio=0 tid=0x00007f064a5b8800 nid=0x884e runnable [0x00007efc06042000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76682878> (a sun.nio.ch.Util$3) - locked <0x00007efd76682860> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76682890> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at kafka.network.Processor.poll(SocketServer.scala:678) at kafka.network.Processor.run(SocketServer.scala:583) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "/config/changes-event-process-thread" #245 prio=5 os_prio=0 tid=0x00007f064aaa3000 nid=0x884d waiting on condition [0x00007efc06143000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678e218> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.common.ZkNodeChangeNotificationListener$ChangeEventProcessThread.doWork(ZkNodeChangeNotificationListener.scala:145) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-request-handler-31" #244 daemon prio=5 os_prio=0 tid=0x00007f064aa98800 nid=0x884c waiting on condition [0x00007efc06542000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-30" #243 daemon prio=5 os_prio=0 tid=0x00007f064aa97000 nid=0x884b runnable [0x00007efc06c59000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-29" #242 daemon prio=5 os_prio=0 tid=0x00007f064aa96000 nid=0x884a runnable [0x00007efc06d5a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-28" #241 daemon prio=5 os_prio=0 tid=0x00007f064aa82000 nid=0x8849 runnable [0x00007efc06e5b000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-27" #240 daemon prio=5 os_prio=0 tid=0x00007f064aa80800 nid=0x8848 runnable [0x00007efc06f5c000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-26" #239 daemon prio=5 os_prio=0 tid=0x00007f064aa7f800 nid=0x8847 runnable [0x00007efc0705d000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-25" #238 daemon prio=5 os_prio=0 tid=0x00007f064aa7e000 nid=0x8846 runnable [0x00007efc0889e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-24" #237 daemon prio=5 os_prio=0 tid=0x00007f064aa7d000 nid=0x8845 runnable [0x00007efc08bdd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-23" #236 daemon prio=5 os_prio=0 tid=0x00007f064aa7c000 nid=0x8844 waiting on condition [0x00007efc09208000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-22" #235 daemon prio=5 os_prio=0 tid=0x00007f064aa7a800 nid=0x8843 runnable [0x00007efc60c84000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-21" #234 daemon prio=5 os_prio=0 tid=0x00007f064aa79800 nid=0x8842 runnable [0x00007efc6225a000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-20" #233 daemon prio=5 os_prio=0 tid=0x00007f064aa78800 nid=0x8841 runnable [0x00007ef6983fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-19" #232 daemon prio=5 os_prio=0 tid=0x00007f064aa77000 nid=0x8840 runnable [0x00007ef6aa1ba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-18" #231 daemon prio=5 os_prio=0 tid=0x00007f064aa76000 nid=0x883f waiting on condition [0x00007ef607476000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efe912c4178> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at kafka.server.AbstractFetcherThread.removePartitions(AbstractFetcherThread.scala:382) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:150) at kafka.server.AbstractFetcherManager$$anonfun$removeFetcherForPartitions$1.apply(AbstractFetcherManager.scala:149) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashMap$$anon$2$$anonfun$foreach$3.apply(HashMap.scala:139) at scala.collection.mutable.HashTable$class.foreachEntry(HashTable.scala:236) at scala.collection.mutable.HashMap.foreachEntry(HashMap.scala:40) at scala.collection.mutable.HashMap$$anon$2.foreach(HashMap.scala:139) at kafka.server.AbstractFetcherManager.removeFetcherForPartitions(AbstractFetcherManager.scala:149) - locked <0x00007efd81c6a208> (a java.lang.Object) at kafka.server.ReplicaManager.makeLeaders(ReplicaManager.scala:1169) at kafka.server.ReplicaManager.becomeLeaderOrFollower(ReplicaManager.scala:1091) - locked <0x00007efd76682ed8> (a java.lang.Object) at kafka.server.KafkaApis.handleLeaderAndIsrRequest(KafkaApis.scala:185) at kafka.server.KafkaApis.handle(KafkaApis.scala:110) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-17" #230 daemon prio=5 os_prio=0 tid=0x00007f064aa75000 nid=0x883e runnable [0x00007ef5f8dfe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-16" #229 daemon prio=5 os_prio=0 tid=0x00007f064aa74000 nid=0x883d runnable [0x00007ef603e76000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-15" #228 daemon prio=5 os_prio=0 tid=0x00007f064aa73000 nid=0x883c runnable [0x00007ef5f03fe000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-14" #227 daemon prio=5 os_prio=0 tid=0x00007f064aa72000 nid=0x883b runnable [0x00007ef6ac10f000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-13" #226 daemon prio=5 os_prio=0 tid=0x00007f064aa71000 nid=0x883a runnable [0x00007ef607f77000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-12" #225 daemon prio=5 os_prio=0 tid=0x00007f064aa70000 nid=0x8839 runnable [0x00007ef532e13000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-11" #224 daemon prio=5 os_prio=0 tid=0x00007f064aa6f000 nid=0x8838 runnable [0x00007ef533c35000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-10" #223 daemon prio=5 os_prio=0 tid=0x00007f064aa6e000 nid=0x8837 runnable [0x00007ef530153000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-9" #222 daemon prio=5 os_prio=0 tid=0x00007f064aa6c800 nid=0x8836 runnable [0x00007ef533f12000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.interrupt(Native Method) at sun.nio.ch.EPollArrayWrapper.interrupt(EPollArrayWrapper.java:317) at sun.nio.ch.EPollSelectorImpl.wakeup(EPollSelectorImpl.java:207) - locked <0x00007efd867b3160> (a java.lang.Object) at org.apache.kafka.common.network.Selector.wakeup(Selector.java:305) at kafka.network.Processor.wakeup(SocketServer.scala:870) at kafka.network.Processor.enqueueResponse(SocketServer.scala:829) at kafka.network.RequestChannel.sendResponse(RequestChannel.scala:336) at kafka.server.KafkaApis.kafka$server$KafkaApis$$sendResponse(KafkaApis.scala:2362) at kafka.server.KafkaApis.sendResponse(KafkaApis.scala:2358) at kafka.server.KafkaApis.kafka$server$KafkaApis$$processResponseCallback$1(KafkaApis.scala:672) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.KafkaApis$$anonfun$handleFetchRequest$4.apply(KafkaApis.scala:688) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:845) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-8" #221 daemon prio=5 os_prio=0 tid=0x00007f064aa6b800 nid=0x8835 runnable [0x00007ef5383b3000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-7" #220 daemon prio=5 os_prio=0 tid=0x00007f064aa6a800 nid=0x8834 waiting on condition [0x00007ef53851b000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd82911860> (a java.util.concurrent.locks.ReentrantLock$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireInterruptibly(AbstractQueuedSynchronizer.java:897) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireInterruptibly(AbstractQueuedSynchronizer.java:1222) at java.util.concurrent.locks.ReentrantLock.lockInterruptibly(ReentrantLock.java:335) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:413) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-6" #219 daemon prio=5 os_prio=0 tid=0x00007f064aa69800 nid=0x8833 runnable [0x00007ef7d5223000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-5" #218 daemon prio=5 os_prio=0 tid=0x00007f064aa68800 nid=0x8832 runnable [0x00007efaf114e000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-4" #217 daemon prio=5 os_prio=0 tid=0x00007f064aa67800 nid=0x8831 runnable [0x00007efbe5bd9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-3" #216 daemon prio=5 os_prio=0 tid=0x00007f064aa66800 nid=0x8830 runnable [0x00007ef530355000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-2" #215 daemon prio=5 os_prio=0 tid=0x00007f064aa66000 nid=0x882f runnable [0x00007ef538248000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-1" #214 daemon prio=5 os_prio=0 tid=0x00007f064aa62800 nid=0x882e runnable [0x00007efae78e1000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e0fe0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418) at kafka.network.RequestChannel.receiveRequest(RequestChannel.scala:342) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:59) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-request-handler-0" #213 daemon prio=5 os_prio=0 tid=0x00007f064aa62000 nid=0x882d runnable [0x00007ef600175000] java.lang.Thread.State: RUNNABLE at kafka.server.ReplicaManager.getLeaderReplicaIfLocal(ReplicaManager.scala:420) at kafka.server.ReplicaManager.kafka$server$ReplicaManager$$read$1(ReplicaManager.scala:896) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:989) at kafka.server.ReplicaManager$$anonfun$readFromLocalLog$1.apply(ReplicaManager.scala:988) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.server.ReplicaManager.readFromLocalLog(ReplicaManager.scala:988) at kafka.server.ReplicaManager.readFromLog$1(ReplicaManager.scala:815) at kafka.server.ReplicaManager.fetchMessages(ReplicaManager.scala:828) at kafka.server.KafkaApis.handleFetchRequest(KafkaApis.scala:680) at kafka.server.KafkaApis.handle(KafkaApis.scala:107) at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:74) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-7" #211 daemon prio=5 os_prio=0 tid=0x00007f064aa57800 nid=0x8828 sleeping[0x00007ef5f02fd000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.ranger.plugin.util.PolicyRefresher.run(PolicyRefresher.java:173) Locked ownable synchronizers: - None "Ranger async Audit cleanup" #209 daemon prio=5 os_prio=0 tid=0x00007f064a20c000 nid=0x8826 waiting on condition [0x00007ef53d7be000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e1f08> (a java.util.concurrent.Semaphore$NonfairSync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.Semaphore.acquire(Semaphore.java:312) at org.apache.ranger.audit.provider.AuditProviderFactory$RangerAsyncAuditCleanup.run(AuditProviderFactory.java:495) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TGT Renewer for ocdp/n20dka0518@ocdp" #207 daemon prio=5 os_prio=0 tid=0x00007f064a20d800 nid=0x8825 waiting on condition [0x00007ef529eb2000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.security.UserGroupInformation$AutoRenewalForUserCredsRunnable.run(UserGroupInformation.java:898) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "TxnMarkerSenderThread-1013" #203 prio=5 os_prio=0 tid=0x00007f0649caf000 nid=0x881d runnable [0x00007ef607375000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd76860aa8> (a sun.nio.ch.Util$3) - locked <0x00007efd76860a90> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd76860a18> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.kafka.common.network.Selector.select(Selector.java:689) at org.apache.kafka.common.network.Selector.poll(Selector.java:409) at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:510) at kafka.common.InterBrokerSendThread.doWork(InterBrokerSendThread.scala:66) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "transaction-log-manager-0" #204 daemon prio=5 os_prio=0 tid=0x00007f0649c8a000 nid=0x881c waiting on condition [0x00007ef5f81fc000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd768502c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "group-metadata-manager-0" #201 daemon prio=5 os_prio=0 tid=0x00007f064a21c000 nid=0x881b waiting on condition [0x00007ef604074000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3ab8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ExpirationReaper-1013-Rebalance" #200 prio=5 os_prio=0 tid=0x00007f064970d800 nid=0x881a runnable [0x00007efbd6d21000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766e21d0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Heartbeat" #199 prio=5 os_prio=0 tid=0x00007f0649df9800 nid=0x8819 runnable [0x00007efbc9270000] java.lang.Thread.State: RUNNABLE at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678fa98> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-topic" #198 prio=5 os_prio=0 tid=0x00007f0649c9a000 nid=0x8818 runnable [0x00007efb885b9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76860e00> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "controller-event-thread" #197 prio=5 os_prio=0 tid=0x00007f064a206800 nid=0x8817 waiting on condition [0x00007ef5f82fd000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd766d3e18> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at kafka.controller.ControllerEventManager$ControllerEventThread.doWork(ControllerEventManager.scala:77) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "LogDirFailureHandler" #196 prio=5 os_prio=0 tid=0x00007f0649df8000 nid=0x8815 waiting on condition [0x00007ef609e78000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd51341ff8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:403) at kafka.server.LogDirFailureChannel.takeNextOfflineLogDir(LogDirFailureChannel.scala:57) at kafka.server.ReplicaManager$LogDirFailureHandler.doWork(ReplicaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-DeleteRecords" #195 prio=5 os_prio=0 tid=0x00007f064a1ef000 nid=0x8814 runnable [0x00007ef530254000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861118> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Fetch" #194 prio=5 os_prio=0 tid=0x00007f064a1ff800 nid=0x8813 runnable [0x00007efafa4d8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861430> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:268) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ExpirationReaper-1013-Produce" #193 prio=5 os_prio=0 tid=0x00007f0649cb2800 nid=0x8812 runnable [0x00007efae56d7000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd7678ffd8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.utils.timer.SystemTimer.advanceClock(Timer.scala:105) at kafka.server.DelayedOperationPurgatory.advanceClock(DelayedOperation.scala:416) at kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper.doWork(DelayedOperation.scala:440) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-socket-acceptor-ListenerName(SASL_PLAINTEXT)-SASL_PLAINTEXT-6667" #192 prio=5 os_prio=0 tid=0x00007f0649cc8000 nid=0x8811 runnable [0x00007efbe74c5000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd766d4130> (a sun.nio.ch.Util$3) - locked <0x00007efd766d4118> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd766d4148> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at kafka.network.Acceptor.run(SocketServer.scala:382) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-kerberos-refresh-thread-ocdp/n20dka0518@ocdp" #191 daemon prio=5 os_prio=0 tid=0x00007f064a1f1000 nid=0x8810 waiting on condition [0x00007efae34b8000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.kafka.common.security.kerberos.KerberosLogin$1.run(KerberosLogin.java:195) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-log-cleaner-thread-0" #190 prio=5 os_prio=0 tid=0x00007f064a1ed000 nid=0x880f waiting on condition [0x00007efc08705000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861758> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedNanos(AbstractQueuedSynchronizer.java:1037) at java.util.concurrent.locks.AbstractQueuedSynchronizer.tryAcquireSharedNanos(AbstractQueuedSynchronizer.java:1328) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:277) at kafka.utils.ShutdownableThread.pause(ShutdownableThread.scala:69) at kafka.log.LogCleaner$CleanerThread.cleanOrSleep(LogCleaner.scala:327) at kafka.log.LogCleaner$CleanerThread.doWork(LogCleaner.scala:289) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "kafka-scheduler-9" #93 daemon prio=5 os_prio=0 tid=0x00007efc30008800 nid=0x8a1a waiting on condition [0x00007efb00172000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-8" #92 daemon prio=5 os_prio=0 tid=0x00007efbb0002000 nid=0x8a19 waiting on condition [0x00007efb0058e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-7" #91 daemon prio=5 os_prio=0 tid=0x00007efb84008000 nid=0x8a16 waiting on condition [0x00007efb141d5000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-6" #90 daemon prio=5 os_prio=0 tid=0x00007efbd0002000 nid=0x8a15 waiting on condition [0x00007efb1c231000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-5" #89 daemon prio=5 os_prio=0 tid=0x00007efb98003000 nid=0x8a14 waiting for monitor entry [0x00007efb1d346000] java.lang.Thread.State: BLOCKED (on object monitor) at kafka.log.Log.minSnapshotsOffsetToRetain(Log.scala:1574) - waiting to lock <0x00007efd59e99790> (a java.lang.Object) at kafka.log.Log.deleteSnapshotsAfterRecoveryPointCheckpoint(Log.scala:1567) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29$$anonfun$apply$31.apply(LogManager.scala:582) at scala.collection.immutable.List.foreach(List.scala:392) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:582) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1$$anonfun$apply$29.apply(LogManager.scala:578) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:578) at kafka.log.LogManager$$anonfun$kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir$1.apply(LogManager.scala:577) at scala.Option.foreach(Option.scala:257) at kafka.log.LogManager.kafka$log$LogManager$$checkpointLogRecoveryOffsetsInDir(LogManager.scala:577) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at kafka.log.LogManager$$anonfun$checkpointLogRecoveryOffsets$1.apply(LogManager.scala:561) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at kafka.log.LogManager.checkpointLogRecoveryOffsets(LogManager.scala:561) at kafka.log.LogManager$$anonfun$startup$3.apply$mcV$sp(LogManager.scala:406) at kafka.utils.KafkaScheduler$$anonfun$1.apply$mcV$sp(KafkaScheduler.scala:114) at kafka.utils.CoreUtils$$anon$1.run(CoreUtils.scala:63) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd506904a0> (a java.util.concurrent.ThreadPoolExecutor$Worker) "kafka-scheduler-4" #88 daemon prio=5 os_prio=0 tid=0x00007efba0002000 nid=0x8a13 waiting on condition [0x00007efb1d7fe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-3" #87 daemon prio=5 os_prio=0 tid=0x00007efc2c00d000 nid=0x8a12 waiting on condition [0x00007efb244b4000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-2" #86 daemon prio=5 os_prio=0 tid=0x00007efb8c002000 nid=0x8a11 waiting on condition [0x00007efb246ae000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-1" #85 daemon prio=5 os_prio=0 tid=0x00007efc10006000 nid=0x8a10 waiting on condition [0x00007efb2607e000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "kafka-scheduler-0" #84 daemon prio=5 os_prio=0 tid=0x00007efbf0004800 nid=0x8a0f waiting on condition [0x00007efb3c1e2000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd506fd8e0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "ThrottledChannelReaper-Request" #35 prio=5 os_prio=0 tid=0x00007f064a1a7800 nid=0x898b waiting on condition [0x00007efc62db8000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd509f7ca0> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Produce" #34 prio=5 os_prio=0 tid=0x00007f064a1a3000 nid=0x898a waiting on condition [0x00007efc62eb9000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50a42090> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "ThrottledChannelReaper-Fetch" #33 prio=5 os_prio=0 tid=0x00007f064a19f800 nid=0x8989 waiting on condition [0x00007efc62fba000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd507b1938> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.DelayQueue.poll(DelayQueue.java:259) at kafka.server.ClientQuotaManager$ThrottledChannelReaper.doWork(ClientQuotaManager.scala:196) at kafka.utils.ShutdownableThread.run(ShutdownableThread.scala:82) Locked ownable synchronizers: - None "SensorExpiryThread" #32 daemon prio=5 os_prio=0 tid=0x00007f064a18a800 nid=0x8988 waiting on condition [0x00007efc630bb000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd50868d70> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Thread-2" #31 daemon prio=5 os_prio=0 tid=0x00007efc580fe000 nid=0x8987 waiting on condition [0x00007efc637bc000] java.lang.Thread.State: TIMED_WAITING (sleeping) at java.lang.Thread.sleep(Native Method) at org.apache.zookeeper.Login$1.run(Login.java:187) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "main-EventThread" #30 daemon prio=5 os_prio=0 tid=0x00007f064a031000 nid=0x8986 waiting on condition [0x00007efc63cfb000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd500184d8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:442) at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:504) Locked ownable synchronizers: - None "main-SendThread(n20dka0502:2181)" #29 daemon prio=5 os_prio=0 tid=0x00007f064a02e800 nid=0x8985 runnable [0x00007efc63dfc000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.EPollArrayWrapper.epollWait(Native Method) at sun.nio.ch.EPollArrayWrapper.poll(EPollArrayWrapper.java:269) at sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:93) at sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:86) - locked <0x00007efd50018c70> (a sun.nio.ch.Util$3) - locked <0x00007efd50018c58> (a java.util.Collections$UnmodifiableSet) - locked <0x00007efd50018be0> (a sun.nio.ch.EPollSelectorImpl) at sun.nio.ch.SelectorImpl.select(SelectorImpl.java:97) at org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:349) at org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1145) Locked ownable synchronizers: - None "metrics-meter-tick-thread-2" #28 daemon prio=5 os_prio=0 tid=0x00007f064a000800 nid=0x8984 runnable [0x00007efc63efd000] java.lang.Thread.State: TIMED_WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:215) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2078) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1093) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-meter-tick-thread-1" #27 daemon prio=5 os_prio=0 tid=0x00007f0649ffd800 nid=0x8983 waiting on condition [0x00007efc63ffe000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd5004df68> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1088) at java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:809) at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "metrics-timeline-scheduled-reporter-1-thread-1" #25 daemon prio=5 os_prio=0 tid=0x00007f0649f31800 nid=0x8982 runnable [0x00007efcb8341000] java.lang.Thread.State: RUNNABLE at java.util.TreeMap.put(TreeMap.java:568) at java.util.AbstractMap.putAll(AbstractMap.java:281) at java.util.TreeMap.putAll(TreeMap.java:327) at org.apache.hadoop.metrics2.sink.timeline.TimelineMetric.addMetricValues(TimelineMetric.java:158) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricWrapper.putMetric(TimelineMetricsCache.java:97) - locked <0x00007eff7e2ce0c8> (a org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricWrapper) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache$TimelineMetricHolder.put(TimelineMetricsCache.java:156) at org.apache.hadoop.metrics2.sink.timeline.cache.TimelineMetricsCache.putTimelineMetric(TimelineMetricsCache.java:194) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheSanitizedTimelineMetric(KafkaTimelineMetricsReporter.java:480) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.cacheKafkaMetered(KafkaTimelineMetricsReporter.java:427) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processMeter(KafkaTimelineMetricsReporter.java:365) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.processMeter(KafkaTimelineMetricsReporter.java:284) at com.yammer.metrics.core.Meter.processWith(Meter.java:131) at org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter.report(KafkaTimelineMetricsReporter.java:319) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter.report(ScheduledReporter.java:185) - locked <0x00007efd50170500> (a org.apache.hadoop.metrics2.sink.kafka.KafkaTimelineMetricsReporter$TimelineScheduledReporter) at org.apache.hadoop.metrics2.sink.kafka.ScheduledReporter$1.run(ScheduledReporter.java:137) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - <0x00007efd50044b48> (a java.util.concurrent.ThreadPoolExecutor$Worker) "RMI TCP Accept-0" #23 daemon prio=5 os_prio=0 tid=0x00007f0649bee000 nid=0x897f runnable [0x00007efcb8d43000] java.lang.Thread.State: RUNNABLE at java.net.PlainSocketImpl.socketAccept(Native Method) at java.net.AbstractPlainSocketImpl.accept(AbstractPlainSocketImpl.java:409) at java.net.ServerSocket.implAccept(ServerSocket.java:545) at java.net.ServerSocket.accept(ServerSocket.java:513) at sun.management.jmxremote.LocalRMIServerSocketFactory$1.accept(LocalRMIServerSocketFactory.java:52) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.executeAcceptLoop(TCPTransport.java:405) at sun.rmi.transport.tcp.TCPTransport$AcceptLoop.run(TCPTransport.java:377) at java.lang.Thread.run(Thread.java:748) Locked ownable synchronizers: - None "Service Thread" #21 daemon prio=9 os_prio=0 tid=0x00007f064972a800 nid=0x897b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread14" #20 daemon prio=9 os_prio=0 tid=0x00007f0649727800 nid=0x897a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread13" #19 daemon prio=9 os_prio=0 tid=0x00007f0649725800 nid=0x8979 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread12" #18 daemon prio=9 os_prio=0 tid=0x00007f0649723000 nid=0x8978 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread11" #17 daemon prio=9 os_prio=0 tid=0x00007f0649721000 nid=0x8977 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C1 CompilerThread10" #16 daemon prio=9 os_prio=0 tid=0x00007f064971f800 nid=0x8976 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread9" #15 daemon prio=9 os_prio=0 tid=0x00007f064971c800 nid=0x8975 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread8" #14 daemon prio=9 os_prio=0 tid=0x00007f064971a800 nid=0x8974 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread7" #13 daemon prio=9 os_prio=0 tid=0x00007f0649718000 nid=0x8973 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread6" #12 daemon prio=9 os_prio=0 tid=0x00007f0649716000 nid=0x8972 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread5" #11 daemon prio=9 os_prio=0 tid=0x00007f0649714000 nid=0x8971 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread4" #10 daemon prio=9 os_prio=0 tid=0x00007f0649711800 nid=0x8970 waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread3" #9 daemon prio=9 os_prio=0 tid=0x00007f0649707800 nid=0x896f waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread2" #8 daemon prio=9 os_prio=0 tid=0x00007f0649705000 nid=0x896e waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread1" #7 daemon prio=9 os_prio=0 tid=0x00007f0649703000 nid=0x896d waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "C2 CompilerThread0" #6 daemon prio=9 os_prio=0 tid=0x00007f0649700800 nid=0x896c waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Signal Dispatcher" #5 daemon prio=9 os_prio=0 tid=0x00007f06496fe800 nid=0x896b runnable [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Surrogate Locker Thread (Concurrent GC)" #4 daemon prio=9 os_prio=0 tid=0x00007f06496fd000 nid=0x896a waiting on condition [0x0000000000000000] java.lang.Thread.State: RUNNABLE Locked ownable synchronizers: - None "Finalizer" #3 daemon prio=8 os_prio=0 tid=0x00007f06496ca000 nid=0x8969 in Object.wait() [0x00007efcce7fe000] java.lang.Thread.State: WAITING (on object monitor) at java.lang.Object.wait(Native Method) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:144) - locked <0x00007efd5004a6a0> (a java.lang.ref.ReferenceQueue$Lock) at java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:165) at java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:216) Locked ownable synchronizers: - None "Reference Handler" #2 daemon prio=10 os_prio=0 tid=0x00007f06496c7800 nid=0x8968 runnable [0x00007efcce8ff000] java.lang.Thread.State: RUNNABLE at sun.nio.ch.FileChannelImpl.unmap0(Native Method) at sun.nio.ch.FileChannelImpl.access$000(FileChannelImpl.java:50) at sun.nio.ch.FileChannelImpl$Unmapper.run(FileChannelImpl.java:820) at sun.misc.Cleaner.clean(Cleaner.java:143) at java.lang.ref.Reference.tryHandlePending(Reference.java:212) at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:153) Locked ownable synchronizers: - None "main" #1 prio=5 os_prio=0 tid=0x00007f0648010000 nid=0x8924 waiting on condition [0x00007f0651707000] java.lang.Thread.State: WAITING (parking) at sun.misc.Unsafe.park(Native Method) - parking to wait for <0x00007efd76861848> (a java.util.concurrent.CountDownLatch$Sync) at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) at java.util.concurrent.CountDownLatch.await(CountDownLatch.java:231) at kafka.server.KafkaServer.awaitShutdown(KafkaServer.scala:639) at kafka.server.KafkaServerStartable.awaitShutdown(KafkaServerStartable.scala:65) at kafka.Kafka$.main(Kafka.scala:76) at kafka.Kafka.main(Kafka.scala) Locked ownable synchronizers: - None "VM Thread" os_prio=0 tid=0x00007f06496be000 nid=0x8967 runnable "Gang worker#0 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648028800 nid=0x8925 runnable "Gang worker#1 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802a800 nid=0x8926 runnable "Gang worker#2 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802c800 nid=0x8927 runnable "Gang worker#3 (Parallel GC Threads)" os_prio=0 tid=0x00007f064802e000 nid=0x8928 runnable "Gang worker#4 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648030000 nid=0x8929 runnable "Gang worker#5 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648032000 nid=0x892a runnable "Gang worker#6 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648033800 nid=0x892b runnable "Gang worker#7 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648035800 nid=0x892c runnable "Gang worker#8 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648037000 nid=0x892d runnable "Gang worker#9 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648039000 nid=0x892e runnable "Gang worker#10 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803b000 nid=0x892f runnable "Gang worker#11 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803c800 nid=0x8930 runnable "Gang worker#12 (Parallel GC Threads)" os_prio=0 tid=0x00007f064803e800 nid=0x8931 runnable "Gang worker#13 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648040800 nid=0x8932 runnable "Gang worker#14 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648042000 nid=0x8933 runnable "Gang worker#15 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648044000 nid=0x8934 runnable "Gang worker#16 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648045800 nid=0x8935 runnable "Gang worker#17 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648047800 nid=0x8936 runnable "Gang worker#18 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648049800 nid=0x8937 runnable "Gang worker#19 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804b000 nid=0x8938 runnable "Gang worker#20 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804d000 nid=0x8939 runnable "Gang worker#21 (Parallel GC Threads)" os_prio=0 tid=0x00007f064804f000 nid=0x893a runnable "Gang worker#22 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648050800 nid=0x893b runnable "Gang worker#23 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648052800 nid=0x893c runnable "Gang worker#24 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648054800 nid=0x893d runnable "Gang worker#25 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648056000 nid=0x893e runnable "Gang worker#26 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648058000 nid=0x893f runnable "Gang worker#27 (Parallel GC Threads)" os_prio=0 tid=0x00007f0648059800 nid=0x8940 runnable "G1 Main Concurrent Mark GC Thread" os_prio=0 tid=0x00007f06480a5000 nid=0x895f runnable "Gang worker#0 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a7000 nid=0x8960 runnable "Gang worker#1 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480a9000 nid=0x8961 runnable "Gang worker#2 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480aa800 nid=0x8962 runnable "Gang worker#3 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ac800 nid=0x8963 runnable "Gang worker#4 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480ae000 nid=0x8964 runnable "Gang worker#5 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b0000 nid=0x8965 runnable "Gang worker#6 (G1 Parallel Marking Threads)" os_prio=0 tid=0x00007f06480b2000 nid=0x8966 runnable "G1 Concurrent Refinement Thread#0" os_prio=0 tid=0x00007f0648091000 nid=0x895e runnable "G1 Concurrent Refinement Thread#1" os_prio=0 tid=0x00007f064808f800 nid=0x895c runnable "G1 Concurrent Refinement Thread#2" os_prio=0 tid=0x00007f064808d800 nid=0x895b runnable "G1 Concurrent Refinement Thread#3" os_prio=0 tid=0x00007f064808b800 nid=0x895a runnable "G1 Concurrent Refinement Thread#4" os_prio=0 tid=0x00007f064808a000 nid=0x8959 runnable "G1 Concurrent Refinement Thread#5" os_prio=0 tid=0x00007f0648088000 nid=0x8958 runnable "G1 Concurrent Refinement Thread#6" os_prio=0 tid=0x00007f0648086000 nid=0x8957 runnable "G1 Concurrent Refinement Thread#7" os_prio=0 tid=0x00007f0648084800 nid=0x8956 runnable "G1 Concurrent Refinement Thread#8" os_prio=0 tid=0x00007f0648082800 nid=0x8955 runnable "G1 Concurrent Refinement Thread#9" os_prio=0 tid=0x00007f0648080800 nid=0x8954 runnable "G1 Concurrent Refinement Thread#10" os_prio=0 tid=0x00007f064807f000 nid=0x8953 runnable "G1 Concurrent Refinement Thread#11" os_prio=0 tid=0x00007f064807d000 nid=0x8952 runnable "G1 Concurrent Refinement Thread#12" os_prio=0 tid=0x00007f064807b000 nid=0x8951 runnable "G1 Concurrent Refinement Thread#13" os_prio=0 tid=0x00007f0648079800 nid=0x8950 runnable "G1 Concurrent Refinement Thread#14" os_prio=0 tid=0x00007f0648077800 nid=0x894f runnable "G1 Concurrent Refinement Thread#15" os_prio=0 tid=0x00007f0648075800 nid=0x894e runnable "G1 Concurrent Refinement Thread#16" os_prio=0 tid=0x00007f0648073800 nid=0x894d runnable "G1 Concurrent Refinement Thread#17" os_prio=0 tid=0x00007f0648072000 nid=0x894c runnable "G1 Concurrent Refinement Thread#18" os_prio=0 tid=0x00007f0648070000 nid=0x894b runnable "G1 Concurrent Refinement Thread#19" os_prio=0 tid=0x00007f064806e000 nid=0x894a runnable "G1 Concurrent Refinement Thread#20" os_prio=0 tid=0x00007f064806c800 nid=0x8949 runnable "G1 Concurrent Refinement Thread#21" os_prio=0 tid=0x00007f064806a800 nid=0x8948 runnable "G1 Concurrent Refinement Thread#22" os_prio=0 tid=0x00007f0648068800 nid=0x8947 runnable "G1 Concurrent Refinement Thread#23" os_prio=0 tid=0x00007f0648067000 nid=0x8946 runnable "G1 Concurrent Refinement Thread#24" os_prio=0 tid=0x00007f0648065000 nid=0x8945 runnable "G1 Concurrent Refinement Thread#25" os_prio=0 tid=0x00007f0648063000 nid=0x8944 runnable "G1 Concurrent Refinement Thread#26" os_prio=0 tid=0x00007f0648061800 nid=0x8943 runnable "G1 Concurrent Refinement Thread#27" os_prio=0 tid=0x00007f064805f800 nid=0x8942 runnable "G1 Concurrent Refinement Thread#28" os_prio=0 tid=0x00007f064805d800 nid=0x8941 runnable "VM Periodic Task Thread" os_prio=0 tid=0x00007f0649c01000 nid=0x8980 waiting on condition JNI global references: 304 ==========